From f3db8c6e47db223958ed6f05d45a6a6650113c60 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:54:36 -0700 Subject: [PATCH 01/26] feat: Add the override marker to the models and mark evaluations Adds an override marker to the FeatureFlag and Segment models. The marker is an attribute of the model type, never part of the JSON representation, so serializing an entity or writing it to a persistent store never carries it. A marked shallow copy is produced with with_override_marker(), which leaves the source entity unmarked and shares its definition. The evaluator marks an evaluation as override-affected when any definition it reads carries the marker: the evaluated flag, a prerequisite at any depth, or a segment consulted during rule matching, including a segment that does not match and a segment referenced by another segment. The marking propagates upward only. A prerequisite's own record reflects the definitions its subtree read, so an unaffected prerequisite inside a marked evaluation is not marked and a marked prerequisite does not mark a sibling. An evaluation that fails with an error reason is still marked when it read an override definition. The marking appears on the evaluation reason as the overrideAffected indicator, present only when true, and on EvalResult.override_affected for the client and event processing. The async evaluator mirrors the sync one. --- ldclient/evaluation.py | 6 + ldclient/impl/async_evaluator.py | 38 ++- ldclient/impl/evaluator.py | 38 ++- ldclient/impl/evaluator_common.py | 16 +- ldclient/impl/model/feature_flag.py | 24 ++ ldclient/impl/model/segment.py | 24 ++ .../impl/test_async_evaluator_overrides.py | 154 +++++++++ .../testing/impl/test_evaluator_overrides.py | 316 ++++++++++++++++++ .../impl/test_model_override_marker.py | 76 +++++ 9 files changed, 681 insertions(+), 11 deletions(-) create mode 100644 ldclient/testing/impl/test_async_evaluator_overrides.py create mode 100644 ldclient/testing/impl/test_evaluator_overrides.py create mode 100644 ldclient/testing/impl/test_model_override_marker.py diff --git a/ldclient/evaluation.py b/ldclient/evaluation.py index 18aa9e29..a65b9c83 100644 --- a/ldclient/evaluation.py +++ b/ldclient/evaluation.py @@ -57,6 +57,12 @@ def reason(self) -> dict: * ``inExperiment``: This is true if the evaluation resulted in an experiment rollout and served one of the variations in the experiment. Otherwise it is false. + * ``overrideAffected``: This is true if a flag override affected the evaluation, directly + or indirectly. It is present only when true. An evaluation is affected when the flag + itself, a prerequisite flag at any depth, or a segment consulted during rule matching + came from the SDK's override store. Flag overrides are currently experimental and + subject to change. + * ``bigSegmentsStatus``: describes the validity of Big Segment information, if and only if the flag evaluation required querying at least one Big Segment; otherwise it returns None. Allowable values are defined in :class:`BigSegmentsStatus`. For more information, read the diff --git a/ldclient/impl/async_evaluator.py b/ldclient/impl/async_evaluator.py index ead01e08..935fc84d 100644 --- a/ldclient/impl/async_evaluator.py +++ b/ldclient/impl/async_evaluator.py @@ -15,7 +15,8 @@ _match_clause_by_kind, _match_single_context_value, _maybe_negate, - check_targets + check_targets, + mark_override_affected ) from ldclient.impl.events.types import EventFactory from ldclient.impl.model import * @@ -56,15 +57,23 @@ def __init__( async def evaluate(self, flag: FeatureFlag, context: Context, event_factory: EventFactory) -> EvalResult: state = EvalResult() state.original_flag_key = flag.key + # Reading the flag's own definition is the first read of the evaluation, so the marking + # starts from the flag's override marker. + state.override_affected = flag.is_override try: state.detail = await self._evaluate(flag, context, state, event_factory) except EvaluationException as e: if self.__logger is not None: self.__logger.error('Could not evaluate flag "%s": %s' % (flag.key, e.message)) state.detail = EvaluationDetail(None, None, {'kind': 'ERROR', 'errorKind': e.error_kind}) + # An evaluation that fails is still marked when it read an override definition. + if state.override_affected: + mark_override_affected(state.detail) return state if state.big_segments_status is not None: state.detail.reason['bigSegmentsStatus'] = state.big_segments_status + if state.override_affected: + mark_override_affected(state.detail) return state async def _evaluate(self, flag: FeatureFlag, context: Context, state: EvalResult, event_factory: EventFactory) -> EvaluationDetail: @@ -120,9 +129,21 @@ async def _check_prerequisites(self, flag: FeatureFlag, context: Context, state: log.warning("Missing prereq flag: " + prereq_key) failed_prereq = prereq else: - state.depth += 1 - prereq_res = await self._evaluate(prereq_flag, context, state, event_factory) - state.depth -= 1 + # The prerequisite is an evaluation in its own right. Its marking starts from + # its own definition and covers only the definitions its subtree reads. The + # parent's marking is restored and merged afterward, so the marking propagates + # upward only. A failure inside the subtree still merges what was read. + parent_override_affected = state.override_affected + state.override_affected = prereq_flag.is_override + try: + state.depth += 1 + prereq_res = await self._evaluate(prereq_flag, context, state, event_factory) + state.depth -= 1 + finally: + prereq_override_affected = state.override_affected + state.override_affected = parent_override_affected or prereq_override_affected + if prereq_override_affected: + mark_override_affected(prereq_res) # Note that if the prerequisite flag is off, we don't consider it a match no matter what its # off variation was. But we still need to evaluate it in order to generate an event. if (not prereq_flag.on) or prereq_res.variation_index != prereq.variation: @@ -146,7 +167,14 @@ async def _clause_matches_context(self, clause: Clause, context: Context, state: if clause.op == 'segmentMatch': for seg_key in clause.values: segment = await self.__get_segment(seg_key) - if segment is not None and await self._segment_matches_context(segment, context, state): + if segment is None: + continue + # The segment definition was read at this point, so an override segment marks the + # evaluation whether or not it matches. A segment that does not match still shapes + # the outcome, for example through a negated clause. + if segment.is_override: + state.override_affected = True + if await self._segment_matches_context(segment, context, state): return _maybe_negate(clause, True) return _maybe_negate(clause, False) diff --git a/ldclient/impl/evaluator.py b/ldclient/impl/evaluator.py index 77a811d4..f6a14a25 100644 --- a/ldclient/impl/evaluator.py +++ b/ldclient/impl/evaluator.py @@ -17,7 +17,8 @@ _maybe_negate, _variation_index_for_context, check_targets, - error_reason + error_reason, + mark_override_affected ) from ldclient.impl.events.types import EventFactory from ldclient.impl.model import * @@ -57,15 +58,23 @@ def __init__( def evaluate(self, flag: FeatureFlag, context: Context, event_factory: EventFactory) -> EvalResult: state = EvalResult() state.original_flag_key = flag.key + # Reading the flag's own definition is the first read of the evaluation, so the marking + # starts from the flag's override marker. + state.override_affected = flag.is_override try: state.detail = self._evaluate(flag, context, state, event_factory) except EvaluationException as e: if self.__logger is not None: self.__logger.error('Could not evaluate flag "%s": %s' % (flag.key, e.message)) state.detail = EvaluationDetail(None, None, {'kind': 'ERROR', 'errorKind': e.error_kind}) + # An evaluation that fails is still marked when it read an override definition. + if state.override_affected: + mark_override_affected(state.detail) return state if state.big_segments_status is not None: state.detail.reason['bigSegmentsStatus'] = state.big_segments_status + if state.override_affected: + mark_override_affected(state.detail) return state def _evaluate(self, flag: FeatureFlag, context: Context, state: EvalResult, event_factory: EventFactory) -> EvaluationDetail: @@ -118,9 +127,21 @@ def _check_prerequisites(self, flag: FeatureFlag, context: Context, state: EvalR log.warning("Missing prereq flag: " + prereq_key) failed_prereq = prereq else: - state.depth += 1 - prereq_res = self._evaluate(prereq_flag, context, state, event_factory) - state.depth -= 1 + # The prerequisite is an evaluation in its own right. Its marking starts from + # its own definition and covers only the definitions its subtree reads. The + # parent's marking is restored and merged afterward, so the marking propagates + # upward only. A failure inside the subtree still merges what was read. + parent_override_affected = state.override_affected + state.override_affected = prereq_flag.is_override + try: + state.depth += 1 + prereq_res = self._evaluate(prereq_flag, context, state, event_factory) + state.depth -= 1 + finally: + prereq_override_affected = state.override_affected + state.override_affected = parent_override_affected or prereq_override_affected + if prereq_override_affected: + mark_override_affected(prereq_res) # Note that if the prerequisite flag is off, we don't consider it a match no matter what its # off variation was. But we still need to evaluate it in order to generate an event. if (not prereq_flag.on) or prereq_res.variation_index != prereq.variation: @@ -144,7 +165,14 @@ def _clause_matches_context(self, clause: Clause, context: Context, state: EvalR if clause.op == 'segmentMatch': for seg_key in clause.values: segment = self.__get_segment(seg_key) - if segment is not None and self._segment_matches_context(segment, context, state): + if segment is None: + continue + # The segment definition was read at this point, so an override segment marks the + # evaluation whether or not it matches. A segment that does not match still shapes + # the outcome, for example through a negated clause. + if segment.is_override: + state.override_affected = True + if self._segment_matches_context(segment, context, state): return _maybe_negate(clause, True) return _maybe_negate(clause, False) diff --git a/ldclient/impl/evaluator_common.py b/ldclient/impl/evaluator_common.py index 04dbebfd..6e0f6a0c 100644 --- a/ldclient/impl/evaluator_common.py +++ b/ldclient/impl/evaluator_common.py @@ -25,7 +25,7 @@ # prerequisite evaluations, and the cached state of any Big Segments query that we may have # ended up having to do for the context. class EvalResult: - __slots__ = ['detail', 'events', 'big_segments_status', 'big_segments_membership', 'original_flag_key', 'prereq_stack', 'segment_stack', 'depth', 'prerequisites'] + __slots__ = ['detail', 'events', 'big_segments_status', 'big_segments_membership', 'original_flag_key', 'prereq_stack', 'segment_stack', 'depth', 'prerequisites', 'override_affected'] def __init__(self): self.detail = None @@ -37,6 +37,11 @@ def __init__(self): self.segment_stack = None # type: Optional[List[str]] self.depth = 0 self.prerequisites = [] # type: List[str] + # True when the evaluation in progress has read a flag or segment definition that carries + # the override marker. While a prerequisite is evaluated, this holds the marking of the + # prerequisite's own subtree. The evaluator merges it into the parent's marking afterward, + # so the marking propagates upward only. + self.override_affected = False def record_prerequisite(self, key: str): if self.depth == 0: @@ -255,3 +260,12 @@ def check_targets(flag: FeatureFlag, context: Context) -> Optional[EvaluationDet def error_reason(error_kind: str) -> dict: return {'kind': 'ERROR', 'errorKind': error_kind} + + +def mark_override_affected(detail: EvaluationDetail) -> None: + """ + Sets the override-affected indicator on the evaluation reason. The indicator is present + only when true, so a reason that was not affected by an override is left untouched and + serializes exactly as before. + """ + detail.reason['overrideAffected'] = True diff --git a/ldclient/impl/model/feature_flag.py b/ldclient/impl/model/feature_flag.py index ae0b6f75..f7c03f4d 100644 --- a/ldclient/impl/model/feature_flag.py +++ b/ldclient/impl/model/feature_flag.py @@ -1,3 +1,4 @@ +import copy from typing import Any, Dict, List, Optional, Set, Union from ldclient.impl.model.clause import Clause @@ -96,10 +97,12 @@ class FeatureFlag(ModelEntity): '_salt', '_track_events', '_debug_events_until_date', + '_is_override', ] def __init__(self, data: dict): super().__init__(data) + self._is_override = False # In the following logic, we're being somewhat lenient in terms of allowing most properties to # be absent even if they are really required in the schema. That's for backward compatibility # with test logic that constructed incomplete JSON, and also with the file data source which @@ -202,3 +205,24 @@ def exclude_from_summaries(self) -> bool: @property def sampling_ratio(self) -> Optional[int]: return self._sampling_ratio + + @property + def is_override(self) -> bool: + """ + True if this flag definition was supplied by an SDK override source rather than by + LaunchDarkly data. The marker is never part of the JSON representation. Only the SDK + components that manage override entries set it. Components that read a definition + through the store may treat a marked definition the same as any other. + + Flag overrides are currently experimental and subject to change. + """ + return self._is_override + + def with_override_marker(self) -> 'FeatureFlag': + """ + Returns a shallow copy of this flag with the override marker set. The copy shares the + underlying data with this instance, which is left unmarked. + """ + marked = copy.copy(self) + marked._is_override = True + return marked diff --git a/ldclient/impl/model/segment.py b/ldclient/impl/model/segment.py index 6ed1b6a8..709518a2 100644 --- a/ldclient/impl/model/segment.py +++ b/ldclient/impl/model/segment.py @@ -1,3 +1,4 @@ +import copy from typing import Any, List, Optional, Set from ldclient.impl.model.attribute_ref import ( @@ -65,10 +66,12 @@ class Segment(ModelEntity): '_unbounded', '_unbounded_context_kind', '_generation', + '_is_override', ] def __init__(self, data: dict): super().__init__(data) + self._is_override = False # In the following logic, we're being somewhat lenient in terms of allowing most properties to # be absent even if they are really required in the schema. That's for backward compatibility # with test logic that constructed incomplete JSON, and also with the file data source which @@ -137,3 +140,24 @@ def unbounded_context_kind(self) -> Optional[str]: @property def generation(self) -> Optional[int]: return self._generation + + @property + def is_override(self) -> bool: + """ + True if this segment definition was supplied by an SDK override source rather than by + LaunchDarkly data. The marker is never part of the JSON representation. Only the SDK + components that manage override entries set it. Components that read a definition + through the store may treat a marked definition the same as any other. + + Flag overrides are currently experimental and subject to change. + """ + return self._is_override + + def with_override_marker(self) -> 'Segment': + """ + Returns a shallow copy of this segment with the override marker set. The copy shares the + underlying data with this instance, which is left unmarked. + """ + marked = copy.copy(self) + marked._is_override = True + return marked diff --git a/ldclient/testing/impl/test_async_evaluator_overrides.py b/ldclient/testing/impl/test_async_evaluator_overrides.py new file mode 100644 index 00000000..e193d3bf --- /dev/null +++ b/ldclient/testing/impl/test_async_evaluator_overrides.py @@ -0,0 +1,154 @@ +""" +Tests for the override marking performed by the async evaluator. These mirror the key +scenarios of the sync evaluator override tests. +""" +import pytest + +from ldclient import Context +from ldclient.impl.events.types import EventFactory +from ldclient.testing.builders import * +from ldclient.testing.impl.test_async_evaluator import AsyncEvaluatorBuilder +from ldclient.testing.impl.test_evaluator_overrides import ( + assert_override_affected, + assert_reason_override_affected, + make_override_test_flag, + prereq_records, + require_prereq_record +) + +override_test_context = Context.create('userkey') +event_factory_with_reasons = EventFactory(True, lambda: 0) + + +@pytest.mark.asyncio +async def test_override_flag_marks_evaluation(): + flag = make_override_test_flag('feature').with_override_marker() + result = await AsyncEvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert_override_affected(True, result) + + +@pytest.mark.asyncio +async def test_plain_flag_is_not_marked(): + flag = make_override_test_flag('feature') + result = await AsyncEvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'FALLTHROUGH'} + assert_override_affected(False, result) + + +@pytest.mark.asyncio +async def test_malformed_override_flag_error_result_is_marked(): + flag = FlagBuilder('feature').on(True).fallthrough_variation(99).variations('off', 'on').build().with_override_marker() + result = await AsyncEvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(True, result) + + +@pytest.mark.asyncio +async def test_prerequisite_cycle_through_an_override_flag_is_marked(): + prereq = make_override_test_flag('prereq', 'feature').with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + evaluator = AsyncEvaluatorBuilder().with_flag(flag).with_flag(prereq).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(True, result) + assert prereq_records(result) == [] + + +@pytest.mark.asyncio +async def test_prerequisite_cycle_below_an_override_flag_keeps_the_flag_marking(): + prereq = make_override_test_flag('prereq', 'feature') + flag = make_override_test_flag('feature', 'prereq').with_override_marker() + evaluator = AsyncEvaluatorBuilder().with_flag(flag).with_flag(prereq).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(True, result) + + +@pytest.mark.asyncio +async def test_override_prerequisite_marks_prerequisite_record_and_top_level(): + prereq = make_override_test_flag('prereq').with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + evaluator = AsyncEvaluatorBuilder().with_flag(prereq).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert_override_affected(True, result) + assert_reason_override_affected(True, require_prereq_record(result, 'prereq').reason) + + +@pytest.mark.asyncio +async def test_override_flag_does_not_mark_unaffected_prerequisite_record(): + prereq = make_override_test_flag('prereq') + flag = make_override_test_flag('feature', 'prereq').with_override_marker() + evaluator = AsyncEvaluatorBuilder().with_flag(prereq).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert_override_affected(True, result) + assert_reason_override_affected(False, require_prereq_record(result, 'prereq').reason) + + +@pytest.mark.asyncio +async def test_unaffected_sibling_prerequisite_record_stays_unmarked(): + s1 = SegmentBuilder('s1').included(override_test_context.key).build() + d = make_override_test_flag('d').with_override_marker() + b = make_override_test_flag('b', 'd') + c = FlagBuilder('c').on(True).variations('off', 'on').fallthrough_variation(0).rules( + FlagRuleBuilder().variation(1).clauses(make_clause_matching_segment_key(s1.key)).build() + ).build() + a = make_override_test_flag('a', 'b', 'c') + evaluator = AsyncEvaluatorBuilder().with_flag(b).with_flag(c).with_flag(d).with_segment(s1).build() + result = await evaluator.evaluate(a, override_test_context, event_factory_with_reasons) + assert_override_affected(True, result) + assert len(prereq_records(result)) == 3 + assert_reason_override_affected(True, require_prereq_record(result, 'd').reason) + assert_reason_override_affected(True, require_prereq_record(result, 'b').reason) + assert_reason_override_affected(False, require_prereq_record(result, 'c').reason) + + +@pytest.mark.asyncio +async def test_override_segment_read_without_matching_marks_evaluation(): + segment = SegmentBuilder('segment').included('someone-else').build().with_override_marker() + flag = make_boolean_flag_matching_segment(segment) + evaluator = AsyncEvaluatorBuilder().with_segment(segment).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert_override_affected(True, result) + + +@pytest.mark.asyncio +async def test_override_segment_referenced_by_segment_rule_marks_evaluation(): + nested = SegmentBuilder('nested-segment').included(override_test_context.key).build().with_override_marker() + outer = SegmentBuilder('outer-segment').rules(SegmentRuleBuilder().clauses(make_clause_matching_segment_key(nested.key)).build()).build() + flag = make_boolean_flag_matching_segment(outer) + evaluator = AsyncEvaluatorBuilder().with_segment(outer).with_segment(nested).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.value is True + assert_override_affected(True, result) + + +@pytest.mark.asyncio +async def test_missing_definitions_do_not_mark_evaluation(): + unrelated = make_override_test_flag('unrelated').with_override_marker() + flag = make_override_test_flag('feature', 'missing') + evaluator = AsyncEvaluatorBuilder().with_flag(unrelated).with_unknown_flag('missing').build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'PREREQUISITE_FAILED', 'prerequisiteKey': 'missing'} + assert_override_affected(False, result) + + segment_flag = make_boolean_flag_with_clauses(make_clause_matching_segment_key('missing')) + evaluator = AsyncEvaluatorBuilder().with_unknown_segment('missing').build() + result = await evaluator.evaluate(segment_flag, override_test_context, event_factory_with_reasons) + assert_override_affected(False, result) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("flag_override,prereq_override", [(False, False), (True, False), (False, True), (True, True)]) +async def test_result_override_affected_matches_reason_indicator(flag_override: bool, prereq_override: bool): + prereq = make_override_test_flag('prereq') + if prereq_override: + prereq = prereq.with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + if flag_override: + flag = flag.with_override_marker() + evaluator = AsyncEvaluatorBuilder().with_flag(prereq).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert_override_affected(flag_override or prereq_override, result) + assert_reason_override_affected(prereq_override, require_prereq_record(result, 'prereq').reason) diff --git a/ldclient/testing/impl/test_evaluator_overrides.py b/ldclient/testing/impl/test_evaluator_overrides.py new file mode 100644 index 00000000..f69bc612 --- /dev/null +++ b/ldclient/testing/impl/test_evaluator_overrides.py @@ -0,0 +1,316 @@ +""" +Tests for the override marking performed by the evaluator. An evaluation is override-affected +when any flag or segment definition it reads carries the override marker. The marking +propagates upward from prerequisites to the flags that depend on them and never sideways or +downward. +""" +from typing import List, Optional + +import pytest + +from ldclient import Context +from ldclient.impl.evaluator_common import EvalResult +from ldclient.impl.events.types import EventFactory, EventInputEvaluation +from ldclient.testing.builders import * +from ldclient.testing.impl.evaluator_util import EvaluatorBuilder + +override_test_context = Context.create('userkey') + +# Events carry their reasons, so a prerequisite record's marking is visible on its reason. +event_factory_with_reasons = EventFactory(True, lambda: 0) + + +def assert_override_affected(expected: bool, result: EvalResult) -> None: + """Checks the reason indicator and the result scalar together. Both report the same marking.""" + assert result.override_affected is expected, "result.override_affected" + assert_reason_override_affected(expected, result.detail.reason) + + +def assert_reason_override_affected(expected: bool, reason: Optional[dict]) -> None: + assert reason is not None + if expected: + assert reason.get('overrideAffected') is True, "reason indicator: %s" % reason + else: + assert 'overrideAffected' not in reason, "reason indicator: %s" % reason + + +def make_override_test_flag(key: str, *prereq_keys: str) -> FeatureFlag: + """A flag that is on and serves variation 1 ("on") by fallthrough, with prerequisites that must each serve variation 1.""" + builder = FlagBuilder(key).on(True).fallthrough_variation(1).off_variation(0).variations('off', 'on') + for prereq_key in prereq_keys: + builder.prerequisite(prereq_key, 1) + return builder.build() + + +def prereq_records(result: EvalResult) -> List[EventInputEvaluation]: + return result.events or [] + + +def require_prereq_record(result: EvalResult, prereq_key: str) -> EventInputEvaluation: + for event in prereq_records(result): + if event.flag is not None and event.flag.key == prereq_key: + return event + raise AssertionError("no record for prerequisite %s" % prereq_key) + + +def test_override_flag_marks_evaluation_when_off(): + flag = FlagBuilder('feature').on(False).off_variation(0).variations('off', 'on').build().with_override_marker() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'OFF' + assert result.detail.value == 'off' + assert_override_affected(True, result) + + +def test_override_flag_marks_evaluation_on_fallthrough(): + flag = make_override_test_flag('feature').with_override_marker() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert result.detail.value == 'on' + assert_override_affected(True, result) + + +def test_override_flag_marks_evaluation_on_rule_match(): + rule = FlagRuleBuilder().id('rule-id').variation(1).clauses(make_clause_matching_context(override_test_context)).build() + flag = FlagBuilder('feature').on(True).fallthrough_variation(0).variations('off', 'on').rules(rule).build().with_override_marker() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'RULE_MATCH' + assert result.detail.reason['ruleId'] == 'rule-id' + assert result.detail.value == 'on' + assert_override_affected(True, result) + + +def test_plain_flag_alone_is_not_marked(): + flag = FlagBuilder('feature').on(False).off_variation(0).variations('off', 'on').build() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'OFF'} + assert_override_affected(False, result) + + +def test_plain_flag_with_prerequisite_and_segment_is_not_marked(): + segment = SegmentBuilder('segment').included(override_test_context.key).build() + prereq = FlagBuilder('prereq').on(True).variations('off', 'on').fallthrough_variation(0).rules( + FlagRuleBuilder().variation(1).clauses(make_clause_matching_segment_key(segment.key)).build() + ).build() + flag = make_override_test_flag('feature', 'prereq') + evaluator = EvaluatorBuilder().with_flag(prereq).with_segment(segment).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'FALLTHROUGH'} + assert result.detail.value == 'on' + assert_override_affected(False, result) + assert_reason_override_affected(False, require_prereq_record(result, 'prereq').reason) + + +def test_malformed_override_flag_error_result_is_marked(): + flag = FlagBuilder('feature').on(True).fallthrough_variation(99).variations('off', 'on').build().with_override_marker() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'ERROR' + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert result.detail.value is None + assert_override_affected(True, result) + + +def test_prerequisite_cycle_through_an_override_flag_is_marked(): + # feature -> prereq -> feature; only prereq is an override + prereq = make_override_test_flag('prereq', 'feature').with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + evaluator = EvaluatorBuilder().with_flag(flag).with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'ERROR' + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(True, result) + assert prereq_records(result) == [] + + +def test_prerequisite_cycle_below_an_override_flag_keeps_the_flag_marking(): + # feature -> prereq -> feature; only feature is an override. The failure happens inside the + # plain prerequisite's subtree, and the top-level marking from the flag's own read survives it. + prereq = make_override_test_flag('prereq', 'feature') + flag = make_override_test_flag('feature', 'prereq').with_override_marker() + evaluator = EvaluatorBuilder().with_flag(flag).with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(True, result) + + +def test_prerequisite_cycle_through_plain_flags_is_not_marked(): + prereq = make_override_test_flag('prereq', 'feature') + flag = make_override_test_flag('feature', 'prereq') + evaluator = EvaluatorBuilder().with_flag(flag).with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(False, result) + + +def test_override_prerequisite_marks_prerequisite_record_and_top_level(): + prereq = make_override_test_flag('prereq').with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + evaluator = EvaluatorBuilder().with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert_override_affected(True, result) + record = require_prereq_record(result, 'prereq') + assert record.reason is not None + assert record.reason['kind'] == 'FALLTHROUGH' + assert_reason_override_affected(True, record.reason) + + +def test_override_flag_does_not_mark_unaffected_prerequisite_record(): + # The marking propagates upward only. The top-level flag's own marker does not leak into + # the record of a prerequisite whose subtree read no override definition. + prereq = make_override_test_flag('prereq') + flag = make_override_test_flag('feature', 'prereq').with_override_marker() + evaluator = EvaluatorBuilder().with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert_override_affected(True, result) + assert_reason_override_affected(False, require_prereq_record(result, 'prereq').reason) + + +def test_override_prerequisite_at_depth_two_marks_all_affected_scopes(): + prereq2 = make_override_test_flag('prereq2').with_override_marker() + prereq1 = make_override_test_flag('prereq1', 'prereq2') + flag = make_override_test_flag('feature', 'prereq1') + evaluator = EvaluatorBuilder().with_flag(prereq1).with_flag(prereq2).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert_override_affected(True, result) + + # The nested record is produced first, during the evaluation of prereq1. + records = prereq_records(result) + assert [r.flag.key for r in records] == ['prereq2', 'prereq1'] + assert_reason_override_affected(True, records[0].reason) + assert_reason_override_affected(True, records[1].reason) + + +def test_unaffected_sibling_prerequisite_record_stays_unmarked(): + # Flag a has prerequisites b and c. Only d, a prerequisite of b, is an override. The marking + # reaches a, b, and d. It does not reach the sibling c, and the plain segments s1 and s2 + # mark nothing. + s1 = SegmentBuilder('s1').included(override_test_context.key).build() + s2 = SegmentBuilder('s2').included(override_test_context.key).build() + d = make_override_test_flag('d').with_override_marker() + b = make_override_test_flag('b', 'd') + c = FlagBuilder('c').on(True).variations('off', 'on').fallthrough_variation(0).rules( + FlagRuleBuilder().variation(1).clauses(make_clause_matching_segment_key(s1.key)).build() + ).build() + a = FlagBuilder('a').on(True).fallthrough_variation(0).off_variation(0).prerequisite('b', 1).prerequisite('c', 1).rules( + FlagRuleBuilder().id('rule-s2').variation(1).clauses(make_clause_matching_segment_key(s2.key)).build() + ).variations('off', 'on').build() + + evaluator = EvaluatorBuilder().with_flag(b).with_flag(c).with_flag(d).with_segment(s1).with_segment(s2).build() + result = evaluator.evaluate(a, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'RULE_MATCH' + assert result.detail.value == 'on' + assert_override_affected(True, result) + + assert len(prereq_records(result)) == 3 + assert_reason_override_affected(True, require_prereq_record(result, 'd').reason) + assert_reason_override_affected(True, require_prereq_record(result, 'b').reason) + assert_reason_override_affected(False, require_prereq_record(result, 'c').reason) + + +def test_override_segment_referenced_by_flag_rule_marks_evaluation(): + segment = SegmentBuilder('segment').included(override_test_context.key).build().with_override_marker() + flag = make_boolean_flag_matching_segment(segment) + evaluator = EvaluatorBuilder().with_segment(segment).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'RULE_MATCH' + assert result.detail.value is True + assert_override_affected(True, result) + + +def test_override_segment_read_without_matching_marks_evaluation(): + # A read is enough to mark the evaluation. The segment does not need to match. + segment = SegmentBuilder('segment').included('someone-else').build().with_override_marker() + flag = make_boolean_flag_matching_segment(segment) + evaluator = EvaluatorBuilder().with_segment(segment).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert result.detail.value is False + assert_override_affected(True, result) + + +def test_override_segment_read_through_negated_clause_marks_evaluation(): + segment = SegmentBuilder('segment').included('someone-else').build().with_override_marker() + flag = make_boolean_flag_with_clauses(negate_clause(make_clause_matching_segment_key(segment.key))) + evaluator = EvaluatorBuilder().with_segment(segment).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'RULE_MATCH' + assert result.detail.value is True + assert_override_affected(True, result) + + +def test_override_segment_referenced_by_segment_rule_marks_evaluation(): + # The outer segment is plain. A rule of the outer segment reads a nested override segment. + nested = SegmentBuilder('nested-segment').included(override_test_context.key).build().with_override_marker() + outer = SegmentBuilder('outer-segment').rules(SegmentRuleBuilder().clauses(make_clause_matching_segment_key(nested.key)).build()).build() + flag = make_boolean_flag_matching_segment(outer) + evaluator = EvaluatorBuilder().with_segment(outer).with_segment(nested).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'RULE_MATCH' + assert result.detail.value is True + assert_override_affected(True, result) + + +def test_override_segment_referenced_by_prerequisite_marks_prerequisite_record_and_top_level(): + segment = SegmentBuilder('segment').included(override_test_context.key).build().with_override_marker() + prereq = FlagBuilder('prereq').on(True).variations('off', 'on').fallthrough_variation(0).rules( + FlagRuleBuilder().variation(1).clauses(make_clause_matching_segment_key(segment.key)).build() + ).build() + flag = make_override_test_flag('feature', 'prereq') + evaluator = EvaluatorBuilder().with_flag(prereq).with_segment(segment).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert_override_affected(True, result) + assert_reason_override_affected(True, require_prereq_record(result, 'prereq').reason) + + +def test_missing_prerequisite_does_not_mark_evaluation(): + # A definition that cannot be resolved contributes nothing, because nothing was read. The + # store holds an unrelated override definition to show that only reads count. + unrelated = make_override_test_flag('unrelated').with_override_marker() + flag = make_override_test_flag('feature', 'missing') + evaluator = EvaluatorBuilder().with_flag(unrelated).with_unknown_flag('missing').build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'PREREQUISITE_FAILED', 'prerequisiteKey': 'missing'} + assert result.detail.value == 'off' + assert_override_affected(False, result) + assert prereq_records(result) == [] + + +def test_missing_segment_does_not_mark_evaluation(): + unrelated = SegmentBuilder('unrelated').included(override_test_context.key).build().with_override_marker() + flag = make_boolean_flag_with_clauses(make_clause_matching_segment_key('missing')) + evaluator = EvaluatorBuilder().with_segment(unrelated).with_unknown_segment('missing').build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'FALLTHROUGH'} + assert result.detail.value is False + assert_override_affected(False, result) + + +@pytest.mark.parametrize("flag_override,prereq_override", [(False, False), (True, False), (False, True), (True, True)]) +def test_result_override_affected_matches_reason_indicator(flag_override: bool, prereq_override: bool): + prereq = make_override_test_flag('prereq') + if prereq_override: + prereq = prereq.with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + if flag_override: + flag = flag.with_override_marker() + evaluator = EvaluatorBuilder().with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert_override_affected(flag_override or prereq_override, result) + assert_reason_override_affected(prereq_override, require_prereq_record(result, 'prereq').reason) + + +def test_marking_does_not_disturb_other_reason_properties(): + segment = SegmentBuilder('segment').unbounded(True).generation(1).build().with_override_marker() + flag = make_boolean_flag_matching_segment(segment) + evaluator = EvaluatorBuilder().with_segment(segment).with_no_big_segments_for_key(override_test_context.key).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'FALLTHROUGH', 'bigSegmentsStatus': 'HEALTHY', 'overrideAffected': True} + + +def test_reason_indicator_is_absent_when_events_do_not_carry_reasons(): + # The marking does not depend on the event factory. The reason on the result is marked either way. + flag = make_override_test_flag('feature').with_override_marker() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, EventFactory(False, lambda: 0)) + assert_override_affected(True, result) diff --git a/ldclient/testing/impl/test_model_override_marker.py b/ldclient/testing/impl/test_model_override_marker.py new file mode 100644 index 00000000..203e0fc5 --- /dev/null +++ b/ldclient/testing/impl/test_model_override_marker.py @@ -0,0 +1,76 @@ +""" +Tests for the override marker carried on the flag and segment models. The marker lives on the +model type and never in the JSON representation. +""" +import json + +import pytest + +from ldclient.impl.model import FeatureFlag, ModelEncoder, Segment +from ldclient.testing.builders import FlagBuilder, SegmentBuilder +from ldclient.versioned_data_kind import FEATURES, SEGMENTS + +flag_data = {'key': 'flag1', 'version': 2, 'on': True, 'variations': [True, False], 'fallthrough': {'variation': 0}, 'salt': 'x'} +segment_data = {'key': 'seg1', 'version': 3, 'included': ['user1'], 'salt': 'y'} + + +@pytest.mark.parametrize("kind,data", [(FEATURES, flag_data), (SEGMENTS, segment_data)]) +def test_marker_defaults_to_false(kind, data): + entity = kind.decode(dict(data)) + assert entity.is_override is False + + +@pytest.mark.parametrize("kind,data", [(FEATURES, flag_data), (SEGMENTS, segment_data)]) +def test_marked_copy_is_marked_and_source_is_not(kind, data): + entity = kind.decode(dict(data)) + marked = entity.with_override_marker() + assert marked.is_override is True + assert entity.is_override is False + assert type(marked) is type(entity) + + +@pytest.mark.parametrize("kind,data", [(FEATURES, flag_data), (SEGMENTS, segment_data)]) +def test_marker_is_never_serialized(kind, data): + entity = kind.decode(dict(data)) + marked = entity.with_override_marker() + assert marked.to_json_dict() == data + assert kind.encode(marked) == data + assert json.loads(ModelEncoder().encode(marked)) == data + assert 'is_override' not in marked.to_json_dict() + assert 'override' not in json.dumps(marked.to_json_dict()).lower() + + +@pytest.mark.parametrize("kind,data", [(FEATURES, flag_data), (SEGMENTS, segment_data)]) +def test_marked_copy_shares_the_definition(kind, data): + entity = kind.decode(dict(data)) + marked = entity.with_override_marker() + assert marked.key == entity.key + assert marked.version == entity.version + assert marked.to_json_dict() is entity.to_json_dict() + assert marked == entity + + +def test_marked_flag_keeps_its_parsed_properties(): + flag = FlagBuilder('flag1').version(7).on(True).variations('a', 'b').fallthrough_variation(1).prerequisite('p', 0).track_events(True).build() + marked = flag.with_override_marker() + assert marked.on is True + assert marked.variations == ['a', 'b'] + assert marked.fallthrough.variation == 1 + assert [p.key for p in marked.prerequisites] == ['p'] + assert marked.track_events is True + assert marked.version == 7 + + +def test_marked_segment_keeps_its_parsed_properties(): + segment = SegmentBuilder('seg1').version(4).included('a').excluded('b').build() + marked = segment.with_override_marker() + assert marked.included == {'a'} + assert marked.excluded == {'b'} + assert marked.version == 4 + + +def test_marking_a_marked_copy_is_a_marked_copy(): + flag = FeatureFlag(dict(flag_data)).with_override_marker() + again = flag.with_override_marker() + assert again.is_override is True + assert again is not flag From 5aa77e5e26ae0c9dd7da0a578ee85964b225df4e Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:11:09 -0700 Subject: [PATCH 02/26] feat: Add the override store, overlay, and data system wiring Adds the override layer described by the OVERRIDE specification: an override store that holds marked flag and segment definitions, an overlay at the store read boundary that returns the override entry for a key in preference to LaunchDarkly data and enumerates the union of both, and a sink that applies each snapshot as a single replacement and notifies flag change listeners of every flag whose evaluation may have changed, including flags that depend on a changed prerequisite or segment. The override source is an option of the FDv2 data system configuration: DataSystemConfig.override_source and ConfigBuilder.overrides(). The source is built with the data system, so invalid configuration raises from the client constructor. It starts before the data system's own threads, so its initial load completes before the constructor returns, and it is closed with the client. OverrideSource and OverrideSink are public protocols so a custom source can be supplied. The client consults the override store before its not-initialized short-circuit. An overridden flag is served before the client has LaunchDarkly data, and a flag absent from the override store still returns the client-not-ready default. The all-flags state reads through the overlay and, while the client has no LaunchDarkly data, contains only the overridden flags. The async client and data system receive the same changes, with override change notifications delivered on the event loop. The OVERRIDE specification test vectors run as a unit test through the full client stack. --- ldclient/async_client.py | 33 +- ldclient/async_config.py | 8 + ldclient/client.py | 35 +- ldclient/config.py | 27 +- ldclient/datasystem.py | 37 +- ldclient/impl/datasystem/__init__.py | 20 + ldclient/impl/datasystem/async_fdv1.py | 5 + ldclient/impl/datasystem/async_fdv2.py | 53 ++- ldclient/impl/datasystem/fdv1.py | 5 + ldclient/impl/datasystem/fdv2.py | 49 ++- ldclient/impl/overrides/__init__.py | 15 + ldclient/impl/overrides/layer.py | 69 ++++ ldclient/impl/overrides/overlay.py | 85 ++++ ldclient/impl/overrides/sink.py | 147 +++++++ ldclient/interfaces.py | 67 ++++ ldclient/testing/impl/test_overrides.py | 376 ++++++++++++++++++ ldclient/testing/mock_components.py | 126 +++++- .../testing/test_async_client_overrides.py | 193 +++++++++ .../testing/test_ldclient_override_vectors.py | 85 ++++ ldclient/testing/test_ldclient_overrides.py | 255 ++++++++++++ .../testdata/override-vectors/vectors.json | 316 +++++++++++++++ 21 files changed, 1992 insertions(+), 14 deletions(-) create mode 100644 ldclient/impl/overrides/__init__.py create mode 100644 ldclient/impl/overrides/layer.py create mode 100644 ldclient/impl/overrides/overlay.py create mode 100644 ldclient/impl/overrides/sink.py create mode 100644 ldclient/testing/impl/test_overrides.py create mode 100644 ldclient/testing/test_async_client_overrides.py create mode 100644 ldclient/testing/test_ldclient_override_vectors.py create mode 100644 ldclient/testing/test_ldclient_overrides.py create mode 100644 ldclient/testing/testdata/override-vectors/vectors.json diff --git a/ldclient/async_client.py b/ldclient/async_client.py index b741aa17..d04f2ab6 100644 --- a/ldclient/async_client.py +++ b/ldclient/async_client.py @@ -95,6 +95,7 @@ def __init__(self, config: AsyncConfig): # these flags, so plain booleans are safe. self._eval_cached_data_warned = False self._all_flags_cached_data_warned = False + self._all_flags_overrides_only_warned = False # Build the object graph here (loop-free). start() supplies the loop-bound # resources: the HTTP session (created lazily), the data source, the @@ -498,17 +499,26 @@ async def _evaluate_internal(self, key: str, context: Context, default: Any, eve if self._config.offline: return EvaluationDetail(default, None, error_reason('CLIENT_NOT_READY')), None + def not_ready() -> Tuple[EvaluationDetail, Optional[FeatureFlag]]: + log.warning("Feature Flag evaluation attempted before client has initialized! Feature store unavailable - returning default: " + str(default) + " for feature key: " + key) + reason = error_reason('CLIENT_NOT_READY') + self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) + return EvaluationDetail(default, None, reason), None + + no_launchdarkly_data = False availability = await self._data_system.data_availability() if availability != DataAvailability.REFRESHED: if availability == DataAvailability.CACHED: if not self._eval_cached_data_warned: self._eval_cached_data_warned = True log.warning("Feature Flag evaluation attempted before client has initialized - using last known values from feature store for feature key: " + key + ". This message is logged once.") + elif self._data_system.override_source_configured: + # No data from LaunchDarkly is available. The store read below still finds an + # entry that the override layer holds, and the SDK serves it. A miss returns the + # not-ready default. + no_launchdarkly_data = True else: - log.warning("Feature Flag evaluation attempted before client has initialized! Feature store unavailable - returning default: " + str(default) + " for feature key: " + key) - reason = error_reason('CLIENT_NOT_READY') - self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) - return EvaluationDetail(default, None, reason), None + return not_ready() if not context.valid: log.warning("Context was invalid for flag evaluation (%s); returning default value" % context.error) @@ -523,6 +533,8 @@ async def _evaluate_internal(self, key: str, context: Context, default: Any, eve self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) return EvaluationDetail(default, None, reason), None if not flag: + if no_launchdarkly_data: + return not_ready() reason = error_reason('FLAG_NOT_FOUND') self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) return EvaluationDetail(default, None, reason), None @@ -572,12 +584,17 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState log.warning("all_flags_state() called, but client is in offline mode. Returning empty state") return FeatureFlagsState(False) + overrides_only = False availability = await self._data_system.data_availability() if availability != DataAvailability.REFRESHED: if availability == DataAvailability.CACHED: if not self._all_flags_cached_data_warned: self._all_flags_cached_data_warned = True log.warning("all_flags_state() called before client has finished initializing! Using last known values from feature store. This message is logged once.") + elif self._data_system.override_source_configured: + # No data from LaunchDarkly is available. The store read below returns only the + # entries that the override layer holds. The result decides the state. + overrides_only = True else: log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state") return FeatureFlagsState(False) @@ -598,6 +615,14 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState log.error("Unable to read flags for all_flag_state: %s" % repr(e)) return FeatureFlagsState(False) + if overrides_only: + if len(flags_map) == 0: + log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state") + return FeatureFlagsState(False) + if not self._all_flags_overrides_only_warned: + self._all_flags_overrides_only_warned = True + log.warning("all_flags_state() called before client has finished initializing! Returning only flags from the override layer. This message is logged once.") + for key, flag in flags_map.items(): if client_only and not flag.get('clientSide', False): continue diff --git a/ldclient/async_config.py b/ldclient/async_config.py index b46b362e..8537d02f 100644 --- a/ldclient/async_config.py +++ b/ldclient/async_config.py @@ -21,6 +21,7 @@ DataSourceBuilder, DataSourceBuilderConfig, HTTPConfig, + OverrideSourceBuilder, PrivateAttributesConfig ) from ldclient.hook import AsyncHook @@ -127,6 +128,13 @@ class AsyncDataSystemConfig: fdv1_fallback_synchronizer: Optional[DataSourceBuilder[AsyncSynchronizer]] = None """An optional fallback synchronizer that will read from FDv1""" + override_source: Optional[OverrideSourceBuilder] = None + """ + An optional override source. Its flag and segment definitions take precedence over + LaunchDarkly data at evaluation time. Flag overrides are currently experimental and subject + to change. + """ + class AsyncConfig(DataSourceBuilderConfig, PrivateAttributesConfig): """Advanced configuration options for the async SDK client. diff --git a/ldclient/client.py b/ldclient/client.py index b4460bc0..f6067723 100644 --- a/ldclient/client.py +++ b/ldclient/client.py @@ -89,6 +89,7 @@ def __init__(self, config: Config, start_wait: float = 5): self._cached_data_warning_lock = threading.Lock() self._eval_cached_data_warned = False self._all_flags_cached_data_warned = False + self._all_flags_overrides_only_warned = False self._owner_pid = os.getpid() self._fork_warned_pids: Dict[int, object] = {} @@ -467,6 +468,13 @@ def _evaluate_internal(self, key: str, context: Context, default: Any, event_fac if self._config.offline: return EvaluationDetail(default, None, error_reason('CLIENT_NOT_READY')), None + def not_ready() -> Tuple[EvaluationDetail, Optional[FeatureFlag]]: + log.warning("Feature Flag evaluation attempted before client has initialized! Feature store unavailable - returning default: " + str(default) + " for feature key: " + key) + reason = error_reason('CLIENT_NOT_READY') + self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) + return EvaluationDetail(default, None, reason), None + + no_launchdarkly_data = False availability = self._data_system.data_availability if availability != DataAvailability.REFRESHED: if availability == DataAvailability.CACHED: @@ -475,11 +483,13 @@ def _evaluate_internal(self, key: str, context: Context, default: Any, event_fac if not self._eval_cached_data_warned: self._eval_cached_data_warned = True log.warning("Feature Flag evaluation attempted before client has initialized - using last known values from feature store for feature key: " + key + ". This message is logged once.") + elif self._data_system.override_source_configured: + # No data from LaunchDarkly is available. The store read below still finds an + # entry that the override layer holds, and the SDK serves it. A miss returns the + # not-ready default. + no_launchdarkly_data = True else: - log.warning("Feature Flag evaluation attempted before client has initialized! Feature store unavailable - returning default: " + str(default) + " for feature key: " + key) - reason = error_reason('CLIENT_NOT_READY') - self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) - return EvaluationDetail(default, None, reason), None + return not_ready() if not context.valid: log.warning("Context was invalid for flag evaluation (%s); returning default value" % context.error) @@ -494,6 +504,8 @@ def _evaluate_internal(self, key: str, context: Context, default: Any, event_fac self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) return EvaluationDetail(default, None, reason), None if not flag: + if no_launchdarkly_data: + return not_ready() reason = error_reason('FLAG_NOT_FOUND') self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) return EvaluationDetail(default, None, reason), None @@ -545,6 +557,7 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: self._check_forked() + overrides_only = False availability = self._data_system.data_availability if availability != DataAvailability.REFRESHED: if availability == DataAvailability.CACHED: @@ -553,6 +566,10 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: if not self._all_flags_cached_data_warned: self._all_flags_cached_data_warned = True log.warning("all_flags_state() called before client has finished initializing! Using last known values from feature store. This message is logged once.") + elif self._data_system.override_source_configured: + # No data from LaunchDarkly is available. The store read below returns only the + # entries that the override layer holds. The result decides the state. + overrides_only = True else: log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state") return FeatureFlagsState(False) @@ -573,6 +590,16 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: log.error("Unable to read flags for all_flag_state: %s" % repr(e)) return FeatureFlagsState(False) + if overrides_only: + if len(flags_map) == 0: + log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state") + return FeatureFlagsState(False) + if not self._all_flags_overrides_only_warned: + with self._cached_data_warning_lock: + if not self._all_flags_overrides_only_warned: + self._all_flags_overrides_only_warned = True + log.warning("all_flags_state() called before client has finished initializing! Returning only flags from the override layer. This message is logged once.") + for key, flag in flags_map.items(): if client_only and not flag.get('clientSide', False): continue diff --git a/ldclient/config.py b/ldclient/config.py index d7ac76a2..b6bb7ff5 100644 --- a/ldclient/config.py +++ b/ldclient/config.py @@ -24,6 +24,7 @@ EventProcessor, FeatureStore, Initializer, + OverrideSource, Synchronizer, UpdateProcessor ) @@ -255,6 +256,23 @@ def build(self, config: DataSourceBuilderConfig) -> T_co: raise NotImplementedError +class OverrideSourceBuilder(Protocol): # pylint: disable=too-few-public-methods + """ + Protocol for building an override source. Flag overrides are currently experimental and + subject to change. + """ + + def build(self, config: DataSourceBuilderConfig) -> OverrideSource: + """ + Builds the override source. Invalid configuration raises an exception, which the client + constructor propagates. + + :param config: the SDK configuration + :return: the built override source + """ + raise NotImplementedError + + @dataclass(frozen=True) class DataSystemConfig: """Configuration for LaunchDarkly's data acquisition strategy.""" @@ -278,6 +296,13 @@ class DataSystemConfig: fdv1_fallback_synchronizer: Optional[DataSourceBuilder[Synchronizer]] = None """An optional fallback synchronizer that will read from FDv1""" + override_source: Optional[OverrideSourceBuilder] = None + """ + An optional override source. Its flag and segment definitions take precedence over + LaunchDarkly data at evaluation time. Flag overrides are currently experimental and subject + to change. + """ + class Config(DataSourceBuilderConfig, PrivateAttributesConfig): """Advanced configuration options for the SDK client. @@ -715,4 +740,4 @@ def _validate(self): log.warning("Missing or blank SDK key") -__all__ = ['Config', 'BigSegmentsConfig', 'DataSourceBuilder', 'DataSystemConfig', 'HTTPConfig'] +__all__ = ['Config', 'BigSegmentsConfig', 'DataSourceBuilder', 'DataSystemConfig', 'HTTPConfig', 'OverrideSourceBuilder'] diff --git a/ldclient/datasystem.py b/ldclient/datasystem.py index d3a61a54..14b95a7e 100644 --- a/ldclient/datasystem.py +++ b/ldclient/datasystem.py @@ -4,7 +4,11 @@ from typing import List, Optional -from ldclient.config import DataSourceBuilder, DataSystemConfig +from ldclient.config import ( + DataSourceBuilder, + DataSystemConfig, + OverrideSourceBuilder +) from ldclient.impl.datasourcev2.polling import ( FallbackToFDv1PollingDataSourceBuilder, PollingDataSourceBuilder @@ -32,6 +36,7 @@ def __init__(self) -> None: self._fdv1_fallback_synchronizer: Optional[DataSourceBuilder[Synchronizer]] = None self._store_mode: DataStoreMode = DataStoreMode.READ_ONLY self._data_store: Optional[FeatureStore] = None + self._override_source: Optional[OverrideSourceBuilder] = None def initializers(self, initializers: Optional[List[DataSourceBuilder[Initializer]]]) -> "ConfigBuilder": """ @@ -80,6 +85,35 @@ def data_store(self, data_store: FeatureStore, store_mode: DataStoreMode) -> "Co self._store_mode = store_mode return self + def overrides(self, source: OverrideSourceBuilder) -> "ConfigBuilder": + """ + Configures the SDK with an override source. Flag overrides are currently experimental + and subject to change. + + The source supplies flag and segment definitions that take precedence over data + received from LaunchDarkly on a per-key basis. Overrides let an operator force one or + more flags to a known state on a running client, whether or not the client can reach + LaunchDarkly. Flags not present in the override data are unaffected. + + The override source is not a data source. It has no effect on the client's + initialization status or data source status. Configuring it changes nothing until the + source actually supplies an override. At most one override source is accepted. + + Example, with the file-based source and the SDK's default data sources: + :: + + from ldclient import datasystem + from ldclient.integrations.overrides import FileOverrideSourceBuilder + + source = FileOverrideSourceBuilder(['/etc/launchdarkly/overrides.json']) + config = Config(sdk_key, datasystem_config=datasystem.default().overrides(source).build()) + + :param source: a builder for the override source, such as + :class:`ldclient.integrations.overrides.FileOverrideSourceBuilder` + """ + self._override_source = source + return self + def build(self) -> DataSystemConfig: """ Builds the data system configuration. @@ -90,6 +124,7 @@ def build(self) -> DataSystemConfig: fdv1_fallback_synchronizer=self._fdv1_fallback_synchronizer, data_store_mode=self._store_mode, data_store=self._data_store, + override_source=self._override_source, ) diff --git a/ldclient/impl/datasystem/__init__.py b/ldclient/impl/datasystem/__init__.py index 26be520d..88f1f7b7 100644 --- a/ldclient/impl/datasystem/__init__.py +++ b/ldclient/impl/datasystem/__init__.py @@ -158,6 +158,17 @@ def environment_id(self) -> Optional[str]: """ raise NotImplementedError + @property + @abstractmethod + def override_source_configured(self) -> bool: + """ + Reports whether the data system was built with an override source. The value is + fixed at construction. When true, the store served by :attr:`store` overlays the + override layer, and the client serves an overridden flag before it has LaunchDarkly + data. + """ + raise NotImplementedError + class AsyncDataSystem(Protocol): """ @@ -232,6 +243,15 @@ def store(self) -> AsyncReadOnlyStore: """ raise NotImplementedError + @property + @abstractmethod + def override_source_configured(self) -> bool: + """ + Reports whether the data system was built with an override source. The value is + fixed at construction. + """ + raise NotImplementedError + class DiagnosticAccumulator(Protocol): def record_stream_init(self, timestamp, duration, failed): diff --git a/ldclient/impl/datasystem/async_fdv1.py b/ldclient/impl/datasystem/async_fdv1.py index 96fbc660..b8b762f1 100644 --- a/ldclient/impl/datasystem/async_fdv1.py +++ b/ldclient/impl/datasystem/async_fdv1.py @@ -124,6 +124,11 @@ async def stop(self): def store(self) -> AsyncReadOnlyStore: return self._store_view + @property + def override_source_configured(self) -> bool: + # Overrides are an option of the FDv2 data system only. + return False + def set_diagnostic_accumulator(self, diagnostic_accumulator: DiagnosticAccumulator): """ Sets the diagnostic accumulator for streaming initialization metrics. diff --git a/ldclient/impl/datasystem/async_fdv2.py b/ldclient/impl/datasystem/async_fdv2.py index 612fe4cb..9f6d93db 100644 --- a/ldclient/impl/datasystem/async_fdv2.py +++ b/ldclient/impl/datasystem/async_fdv2.py @@ -38,6 +38,11 @@ ) from ldclient.impl.datasystem.store import _decode from ldclient.impl.listeners import Listeners +from ldclient.impl.overrides import ( + AsyncOverrideStoreView, + OverrideLayer, + OverrideSinkImpl +) from ldclient.impl.util import _LD_FD_FALLBACK_HEADER, _Fail, log from ldclient.interfaces import ( AsyncFeatureStore, @@ -47,7 +52,9 @@ DataSourceErrorKind, DataSourceState, DataStoreMode, - DataStoreStatus + DataStoreStatus, + FlagChange, + OverrideSource ) from ldclient.versioned_data_kind import VersionedDataKind @@ -270,6 +277,17 @@ def __init__( self._store_view = _AsyncReadOnlyStoreView(self._store) + # Set only when an override source is configured. See the sync FDv2 for the roles of + # the layer, the overlay, and the source. Building the source here makes an invalid + # source configuration a construction error. + self._override_source: Optional[OverrideSource] = None + self._override_layer: Optional[OverrideLayer] = None + self._overlay: Optional[AsyncOverrideStoreView] = None + if data_system_config.override_source is not None and not self._disabled: + self._override_source = data_system_config.override_source.build(config) + self._override_layer = OverrideLayer() + self._overlay = AsyncOverrideStoreView(self._store_view, self._override_layer) + # Concurrency self._stop_event = AsyncEvent() self._lock = AsyncLock() @@ -314,11 +332,33 @@ def start(self, set_on_ready: AsyncEvent): self._stop_event.clear() + if self._override_source is not None and self._override_layer is not None: + # The source runs on its own threads. Flag change notifications are delivered on + # the event loop, like every other change notification of the async client. The + # merged view for the change computation reads the in-memory store, which is the + # only store that can be read synchronously. While a persistent store is the + # active store, the dependency fan-out is limited to the in-memory data, and the + # directly changed keys are still notified. + loop = asyncio.get_running_loop() + listeners = self._flag_change_listeners + + def notify(key: str) -> None: + loop.call_soon_threadsafe(listeners.notify, FlagChange(key)) + + sink = OverrideSinkImpl(self._override_layer, self._store._memory_store, notify, listeners.has_listeners) + self._override_source.start(sink) + # Start the main coordination loop self._runner.spawn("AsyncFDv2-main", lambda: self._run_main_loop(set_on_ready)) async def stop(self): """Stop the AsyncFDv2 data system and all the work it is coordinating.""" + if self._override_source is not None: + try: + self._override_source.close() + except Exception as e: + log.error("Error closing the override source: %s", e) + self._stop_event.set() async with self._lock: @@ -628,9 +668,18 @@ async def reader(): @property def store(self) -> AsyncReadOnlyStore: - """Get the underlying store for flag evaluation.""" + """ + Get the store for flag evaluation. When an override source is configured, this is the + overlay that serves override entries in preference to LaunchDarkly data. + """ + if self._overlay is not None: + return self._overlay return self._store_view + @property + def override_source_configured(self) -> bool: + return self._override_source is not None + async def data_availability(self) -> DataAvailability: # type: ignore[override] """Reports what form of data is currently available, awaiting the store's readiness so a persistent store populated by another process is recognized diff --git a/ldclient/impl/datasystem/fdv1.py b/ldclient/impl/datasystem/fdv1.py index 09ca6939..dabaa12d 100644 --- a/ldclient/impl/datasystem/fdv1.py +++ b/ldclient/impl/datasystem/fdv1.py @@ -233,6 +233,11 @@ def stop(self): def store(self) -> ReadOnlyStore: return self._store_view + @property + def override_source_configured(self) -> bool: + # Overrides are an option of the FDv2 data system only. + return False + @property def environment_id(self) -> Optional[str]: return self._data_source_update_sink.environment_id diff --git a/ldclient/impl/datasystem/fdv2.py b/ldclient/impl/datasystem/fdv2.py index 75f8d28c..4d5398fb 100644 --- a/ldclient/impl/datasystem/fdv2.py +++ b/ldclient/impl/datasystem/fdv2.py @@ -16,6 +16,11 @@ ) from ldclient.impl.datasystem.store import Store, _decode from ldclient.impl.listeners import Listeners +from ldclient.impl.overrides import ( + OverrideLayer, + OverrideSinkImpl, + OverrideStoreView +) from ldclient.impl.repeating_task import RepeatingTask from ldclient.impl.rwlock import ReadWriteLock from ldclient.impl.util import _LD_FD_FALLBACK_HEADER, _Fail, log @@ -26,6 +31,8 @@ DataStoreMode, DataStoreStatus, FeatureStore, + FlagChange, + OverrideSource, ReadOnlyStore, Synchronizer ) @@ -257,6 +264,20 @@ def __init__( self._store_view = _ReadOnlyStoreView(self._store) + # The following are set only when an override source is configured. The layer holds the + # override entries. The overlay serves them in preference to the store's data at the + # store read boundary. The source populates the layer at runtime. The override system + # has no effect on initialization status or data availability. Building the source + # here makes an invalid source configuration a construction error, like any other + # invalid component configuration. + self._override_source: Optional[OverrideSource] = None + self._override_layer: Optional[OverrideLayer] = None + self._overlay: Optional[OverrideStoreView] = None + if data_system_config.override_source is not None and not self._disabled: + self._override_source = data_system_config.override_source.build(config) + self._override_layer = OverrideLayer() + self._overlay = OverrideStoreView(self._store_view, self._override_layer) + def _create_store(self, flag_change_listeners: Listeners, change_set_listeners: Listeners) -> Store: return Store(flag_change_listeners, change_set_listeners) @@ -288,6 +309,17 @@ def start(self, set_on_ready: Event): self._stop_event.clear() + if self._override_source is not None and self._override_layer is not None: + # The source starts before the main thread, so a source that loads synchronously + # has its overrides in place before the client begins evaluating. + sink = OverrideSinkImpl( + self._override_layer, + self._store_view, + lambda key: self._flag_change_listeners.notify(FlagChange(key)), + self._flag_change_listeners.has_listeners, + ) + self._override_source.start(sink) + # Start the main coordination thread main_thread = Thread( target=self._run_main_loop, @@ -300,6 +332,12 @@ def start(self, set_on_ready: Event): def stop(self): """Stop the FDv2 data system and all associated threads.""" + if self._override_source is not None: + try: + self._override_source.close() + except Exception as e: + log.error("Error closing the override source: %s", e) + self._stop_event.set() with self._lock.write(): @@ -633,9 +671,18 @@ def environment_id(self) -> Optional[str]: @property def store(self) -> ReadOnlyStore: - """Get the underlying store for flag evaluation.""" + """ + Get the store for flag evaluation. When an override source is configured, this is the + overlay that serves override entries in preference to LaunchDarkly data. + """ + if self._overlay is not None: + return self._overlay return self._store_view + @property + def override_source_configured(self) -> bool: + return self._override_source is not None + __all__ = [ 'ConditionDirective', diff --git a/ldclient/impl/overrides/__init__.py b/ldclient/impl/overrides/__init__.py new file mode 100644 index 00000000..b3e7126e --- /dev/null +++ b/ldclient/impl/overrides/__init__.py @@ -0,0 +1,15 @@ +""" +The flag and segment override layer. The layer is a runtime-mutable collection of flag and +segment definitions supplied by an override source. Those definitions take precedence over +LaunchDarkly data at evaluation time. Flag overrides are currently experimental and subject to +change. +""" + +from ldclient.impl.overrides.layer import OverrideLayer +from ldclient.impl.overrides.overlay import ( + AsyncOverrideStoreView, + OverrideStoreView +) +from ldclient.impl.overrides.sink import OverrideSinkImpl + +__all__ = ['AsyncOverrideStoreView', 'OverrideLayer', 'OverrideSinkImpl', 'OverrideStoreView'] diff --git a/ldclient/impl/overrides/layer.py b/ldclient/impl/overrides/layer.py new file mode 100644 index 00000000..d03dc7af --- /dev/null +++ b/ldclient/impl/overrides/layer.py @@ -0,0 +1,69 @@ +""" +The override store. It holds the entries an override source has loaded, keyed by flag key or +segment key, and is replaced wholesale on each update. +""" + +from typing import Any, Dict, Mapping, Optional, Tuple + +from ldclient.impl.model import ModelEntity +from ldclient.impl.rwlock import ReadWriteLock +from ldclient.versioned_data_kind import FEATURES, SEGMENTS, VersionedDataKind + +LayerContents = Dict[VersionedDataKind, Dict[str, ModelEntity]] + + +def _empty_contents() -> LayerContents: + return {FEATURES: {}, SEGMENTS: {}} + + +class OverrideLayer: + """ + A thread-safe store of override entries, replaced wholesale on each update from an override + source. Each entry is a marked shallow copy of the definition the source supplied, so the + source's own objects are never marked and a source may retain and resupply them. + """ + + def __init__(self): + self._lock = ReadWriteLock() + self._contents: LayerContents = _empty_contents() + # Read without the lock on every evaluation. A single attribute read is atomic, so the + # per-evaluation cost of a configured but unpopulated override layer is negligible. + self._non_empty = False + + def set_all(self, flags: Mapping[str, Any], segments: Mapping[str, Any]) -> Tuple[LayerContents, LayerContents]: + """ + Atomically replaces the entire layer contents. Empty mappings clear the layer. + + Values may be model objects or their JSON dictionary form. A dictionary is decoded with + the model constructor, which raises ``ValueError`` for an invalid definition, and nothing + is replaced in that case. + + :return: the previous and the new contents. The returned dictionaries must not be modified. + """ + replacement: LayerContents = { + FEATURES: {key: FEATURES.decode(item).with_override_marker() for key, item in flags.items()}, + SEGMENTS: {key: SEGMENTS.decode(item).with_override_marker() for key, item in segments.items()}, + } + count = len(replacement[FEATURES]) + len(replacement[SEGMENTS]) + with self._lock.write(): + previous = self._contents + self._contents = replacement + self._non_empty = count > 0 + return previous, replacement + + def get(self, kind: VersionedDataKind, key: str) -> Optional[ModelEntity]: + """Returns the override entry for a key, or None when the layer has none.""" + if not self._non_empty: + return None + with self._lock.read(): + return self._contents.get(kind, {}).get(key) + + def all(self, kind: VersionedDataKind) -> Dict[str, ModelEntity]: + """Returns the entries of the given kind. The returned dictionary must not be modified.""" + with self._lock.read(): + return self._contents.get(kind, {}) + + @property + def is_empty(self) -> bool: + """True when the layer holds no entries.""" + return not self._non_empty diff --git a/ldclient/impl/overrides/overlay.py b/ldclient/impl/overrides/overlay.py new file mode 100644 index 00000000..685352d6 --- /dev/null +++ b/ldclient/impl/overrides/overlay.py @@ -0,0 +1,85 @@ +""" +The overlay at the store read boundary. A read for a key returns the override entry when one +exists and the LaunchDarkly entry otherwise. Targeting rules, prerequisites, and segment matches +behave identically for overridden and ordinary data, because they are the same reads through +the same boundary. +""" + +from typing import Any, Callable, Dict, Optional + +from ldclient.impl.overrides.layer import OverrideLayer +from ldclient.interfaces import AsyncReadOnlyStore, ReadOnlyStore +from ldclient.versioned_data_kind import VersionedDataKind + + +def _merge_all(base_items: Optional[Dict[str, Any]], override_items: Dict[str, Any]) -> Dict[str, Any]: + """The union of the base items and the override items. The override entry wins for a key present in both.""" + if base_items is None: + return dict(override_items) + if len(override_items) == 0: + return base_items + merged = dict(base_items) + merged.update(override_items) + return merged + + +class OverrideStoreView(ReadOnlyStore): + """ + Merges an override layer over a base read-only store. + + A per-key read serves an override entry whatever the state of the base store, so an + uninitialized base still serves overrides. An enumeration is the union of the base items + and the layer's items, and the override entry wins for a key present in both, including a + key the base holds as a deleted item. When the base enumeration fails and the layer holds + entries, the layer's entries alone are returned. When the layer is empty, the failure is + raised as before. + """ + + def __init__(self, base: ReadOnlyStore, layer: OverrideLayer): + self._base = base + self._layer = layer + + def get(self, kind: VersionedDataKind, key: str, callback: Callable[[Any], Any] = lambda x: x) -> Any: + item = self._layer.get(kind, key) + if item is not None: + return callback(item) + return self._base.get(kind, key, callback) + + def all(self, kind: VersionedDataKind, callback: Callable[[Any], Any] = lambda x: x) -> Any: + override_items = self._layer.all(kind) + try: + base_items = self._base.all(kind, lambda x: x) + except Exception: + if len(override_items) == 0: + raise + base_items = None + return callback(_merge_all(base_items, override_items)) + + @property + def initialized(self) -> bool: + # The override layer never affects initialization status or data availability. + return self._base.initialized + + +class AsyncOverrideStoreView(AsyncReadOnlyStore): + """The async counterpart of :class:`OverrideStoreView`, over an async base store.""" + + def __init__(self, base: AsyncReadOnlyStore, layer: OverrideLayer): + self._base = base + self._layer = layer + + async def get(self, kind: VersionedDataKind, key: str) -> Optional[Any]: + item = self._layer.get(kind, key) + if item is not None: + return item + return await self._base.get(kind, key) + + async def all(self, kind: VersionedDataKind) -> Dict[str, Any]: + override_items = self._layer.all(kind) + try: + base_items: Optional[Dict[str, Any]] = await self._base.all(kind) + except Exception: + if len(override_items) == 0: + raise + base_items = None + return _merge_all(base_items, override_items) diff --git a/ldclient/impl/overrides/sink.py b/ldclient/impl/overrides/sink.py new file mode 100644 index 00000000..03866dc9 --- /dev/null +++ b/ldclient/impl/overrides/sink.py @@ -0,0 +1,147 @@ +""" +The sink that applies override snapshots to the layer and notifies flag change listeners of the +flags whose evaluation may have changed. +""" + +import threading +from typing import Any, Callable, Dict, Optional, Set + +from ldclient.impl.dependency_tracker import DependencyTracker, KindAndKey +from ldclient.impl.overrides.layer import LayerContents, OverrideLayer +from ldclient.impl.util import log +from ldclient.interfaces import OverrideSink, ReadOnlyStore +from ldclient.versioned_data_kind import FEATURES, SEGMENTS, VersionedDataKind + +MergedView = Dict[VersionedDataKind, Dict[str, Any]] + +_DIFF_KINDS = (FEATURES, SEGMENTS) + + +class OverrideSinkImpl(OverrideSink): + """ + Applies override layer replacements supplied by an override source, then notifies flag + change listeners of every flag whose merged-view evaluation may have changed. Calls are + serialized, so overlapping updates from a source cannot interleave. + """ + + def __init__( + self, + layer: OverrideLayer, + base: ReadOnlyStore, + notify: Callable[[str], None], + has_listeners: Callable[[], bool], + ): + """ + :param layer: the override layer to write to + :param base: the store holding LaunchDarkly data, without the overlay. Merged-view + snapshots for change computation are built from it plus the layer. + :param notify: receives the key of each affected flag + :param has_listeners: reports whether anything listens for flag changes, so the change + computation can be skipped when nothing does + """ + self._layer = layer + self._base = base + self._notify = notify + self._has_listeners = has_listeners + self._lock = threading.Lock() + + def set_overrides(self, flags: Dict[str, Any], segments: Dict[str, Any]) -> None: # type: ignore[override] + with self._lock: + if not self._has_listeners(): + self._layer.set_all(flags, segments) + return + + previous, current = self._layer.set_all(flags, segments) + old_merged = snapshot_merged_view(self._base, previous) + new_merged = snapshot_merged_view(self._base, current) + affected = compute_affected_flags(previous, current, old_merged, new_merged) + if len(affected) > 0: + log.debug("Override update affected %d flag(s)", len(affected)) + for key in sorted(affected): + self._notify(key) + + +def snapshot_merged_view(base: ReadOnlyStore, overrides: LayerContents) -> MergedView: + """ + Captures the merged view of a base store and a layer snapshot: base data with the override + entries overlaid. A base read failure for a kind yields just the overrides for that kind. + This degrades the dependency fan-out but never loses the directly changed keys. + """ + view: MergedView = {} + for kind in _DIFF_KINDS: + items: Dict[str, Any] = {} + try: + base_items = base.all(kind, lambda x: x) + if base_items is not None: + items.update(base_items) + except Exception as e: + log.debug("Unable to read %s for override change computation: %s", kind.namespace, e) + items.update(overrides.get(kind, {})) + view[kind] = items + return view + + +def compute_affected_flags( + old_overrides: LayerContents, + new_overrides: LayerContents, + old_merged: MergedView, + new_merged: MergedView, +) -> Set[str]: + """ + Returns the keys of all flags whose merged-view evaluation may have changed when the + override layer was replaced. The result includes the flags whose override entries were + added, removed, or changed. Dependency fan-out adds every flag that depends, directly or + transitively, on any added, removed, or changed entry of either kind. + """ + seeds = diff_overrides(old_overrides, new_overrides) + if len(seeds) == 0: + return set() + + # Dependency edges are computed over both the old and the new merged views, because a + # replacement can rewire dependencies. For example, removing a flag override restores the + # prerequisite edges of the LaunchDarkly definition. Flags that depended on the override's + # references exist as dependents only in the old view. + old_tracker = _tracker_from_view(old_merged) + new_tracker = _tracker_from_view(new_merged) + affected: Set[KindAndKey] = set() + for seed in seeds: + old_tracker.add_affected_items(affected, seed) + new_tracker.add_affected_items(affected, seed) + return {item.key for item in affected if item.kind == FEATURES} + + +def diff_overrides(old_overrides: LayerContents, new_overrides: LayerContents) -> Set[KindAndKey]: + """ + Returns a key for each entry whose override differs between the two layer snapshots. An + added or removed entry is always a change, even when its content matches the underlying + LaunchDarkly data, because the override marker alone changes the served entry. Entries + present in both snapshots are compared by version and definition. The layer is rebuilt + wholesale on every update, so identity comparison would report every retained entry as + changed. + """ + seeds: Set[KindAndKey] = set() + for kind in _DIFF_KINDS: + old_items = old_overrides.get(kind, {}) + new_items = new_overrides.get(kind, {}) + for key, old_item in old_items.items(): + new_item = new_items.get(key) + if new_item is None or not _items_equal(old_item, new_item): + seeds.add(KindAndKey(kind=kind, key=key)) + for key in new_items: + if key not in old_items: + seeds.add(KindAndKey(kind=kind, key=key)) + return seeds + + +def _items_equal(a: Any, b: Any) -> bool: + if a.version != b.version: + return False + return a.to_json_dict() == b.to_json_dict() + + +def _tracker_from_view(view: MergedView) -> DependencyTracker: + tracker = DependencyTracker() + for kind in _DIFF_KINDS: + for key, item in view.get(kind, {}).items(): + tracker.update_dependencies_from(kind, key, item) + return tracker diff --git a/ldclient/interfaces.py b/ldclient/interfaces.py index 2c9c245f..90e6e549 100644 --- a/ldclient/interfaces.py +++ b/ldclient/interfaces.py @@ -2128,6 +2128,73 @@ def stop(self): raise NotImplementedError +class OverrideSink(Protocol): # pylint: disable=too-few-public-methods + """ + OverrideSink receives the contents of the SDK's flag and segment override layer. The SDK + implements it and passes it to an :class:`OverrideSource` when the source starts. Override + sources call it, they do not implement it. + + Flag overrides are currently experimental and subject to change. + """ + + @abstractmethod + def set_overrides(self, flags: Mapping[str, Any], segments: Mapping[str, Any]) -> None: + """ + Replaces the entire override layer with the given flag and segment definitions. Each call + is a full snapshot: entries absent from the call are removed from the layer, and empty + mappings clear the layer. The new contents are visible to evaluations when the call + returns. + + The mappings are keyed by flag key and segment key. Each value is a full definition, + either a :class:`ldclient.impl.model.FeatureFlag` or :class:`ldclient.impl.model.Segment` + model object or its JSON dictionary form. A dictionary that is not a valid definition + raises ``ValueError``. The SDK itself marks the entries as overrides. + + This method is safe to call from any thread. Calls are serialized by the SDK. + + :param flags: the flag definitions, keyed by flag key + :param segments: the segment definitions, keyed by segment key + """ + raise NotImplementedError + + +class OverrideSource(Protocol): + """ + OverrideSource supplies flag and segment overrides that take precedence over LaunchDarkly data + at evaluation time, on a per-key basis. Overrides exist for resilience during an incident. + They let an operator force one or more flags to a known state on a running client, whether + or not the client can reach LaunchDarkly. + + An override source is not a data source. It does not participate in the data system's + initializer and synchronizer pipeline. The override layer it populates has no effect on the + client's initialization status, data availability, or data source status. + + To configure an override source, use :meth:`ldclient.datasystem.ConfigBuilder.overrides`. + + Flag overrides are currently experimental and subject to change. + """ + + @abstractmethod + def start(self, sink: OverrideSink) -> None: + """ + Begins supplying overrides to the sink and returns without blocking on long-running + work. Implementations typically perform an initial load synchronously, then push a full + replacement snapshot to the sink whenever their backing data changes, until + :meth:`close` is called. A failed load leaves the previously supplied layer untouched by + not calling the sink. + + :param sink: the sink that receives each snapshot + """ + raise NotImplementedError + + @abstractmethod + def close(self) -> None: + """ + Stops the source and releases any resources it holds. + """ + raise NotImplementedError + + class AsyncInitializer(Protocol): # pylint: disable=too-few-public-methods """ AsyncInitializer represents a component capable of retrieving a single data result diff --git a/ldclient/testing/impl/test_overrides.py b/ldclient/testing/impl/test_overrides.py new file mode 100644 index 00000000..6da55c2d --- /dev/null +++ b/ldclient/testing/impl/test_overrides.py @@ -0,0 +1,376 @@ +""" +Tests for the override layer, the overlay at the store read boundary, and the sink that applies +snapshots and notifies flag change listeners. +""" +from typing import Any, Dict, List, Optional + +import pytest + +from ldclient.impl.datasystem.store import InMemoryFeatureStore +from ldclient.impl.overrides import ( + AsyncOverrideStoreView, + OverrideLayer, + OverrideSinkImpl, + OverrideStoreView +) +from ldclient.testing.builders import ( + FlagBuilder, + FlagRuleBuilder, + SegmentBuilder, + make_clause_matching_segment_key +) +from ldclient.versioned_data_kind import FEATURES, SEGMENTS + + +class FakeBaseStore: + """A minimal read-only store with controllable contents, initialization state, and failures.""" + + def __init__(self, flags: Optional[Dict[str, Any]] = None, segments: Optional[Dict[str, Any]] = None, initialized: bool = True): + self.flags = dict(flags or {}) + self.segments = dict(segments or {}) + self._initialized = initialized + self.fail_all = False + + def _items(self, kind): + return self.flags if kind == FEATURES else self.segments + + def get(self, kind, key, callback=lambda x: x): + return callback(self._items(kind).get(key)) + + def all(self, kind, callback=lambda x: x): + if self.fail_all: + raise RuntimeError("store failure") + return callback(dict(self._items(kind))) + + @property + def initialized(self) -> bool: + return self._initialized + + +class FakeAsyncBaseStore: + def __init__(self, base: FakeBaseStore): + self._base = base + + async def get(self, kind, key): + return self._base.get(kind, key) + + async def all(self, kind): + return self._base.all(kind) + + +def flag(key: str, version: int = 1): + return FlagBuilder(key).version(version).on(True).variations(False, True).fallthrough_variation(1).off_variation(0).build() + + +def segment(key: str, version: int = 1): + return SegmentBuilder(key).version(version).build() + + +# --------------------------------------------------------------------------- +# Layer +# --------------------------------------------------------------------------- + +def test_layer_marks_copies_without_mutating_source(): + layer = OverrideLayer() + f = flag('flag1', 2) + s = segment('segment1', 3) + layer.set_all({'flag1': f}, {'segment1': s}) + + assert f.is_override is False + assert s.is_override is False + stored_flag = layer.get(FEATURES, 'flag1') + assert stored_flag is not None + assert stored_flag.is_override is True + assert stored_flag.version == 2 + stored_segment = layer.get(SEGMENTS, 'segment1') + assert stored_segment is not None + assert stored_segment.is_override is True + + +def test_layer_decodes_dictionaries_and_marks_them(): + layer = OverrideLayer() + layer.set_all({'flag1': flag('flag1').to_json_dict()}, {'segment1': segment('segment1').to_json_dict()}) + stored_flag = layer.get(FEATURES, 'flag1') + assert stored_flag is not None + assert stored_flag.is_override is True + assert stored_flag.on is True + stored_segment = layer.get(SEGMENTS, 'segment1') + assert stored_segment is not None + assert stored_segment.is_override is True + + +def test_layer_rejects_invalid_definitions_and_keeps_its_contents(): + layer = OverrideLayer() + layer.set_all({'flag1': flag('flag1')}, {}) + with pytest.raises(ValueError): + layer.set_all({'flag2': {'key': 'flag2', 'version': 'not a number'}}, {}) + assert layer.get(FEATURES, 'flag1') is not None + assert layer.get(FEATURES, 'flag2') is None + with pytest.raises(ValueError): + layer.set_all({}, {'seg': {'key': 'seg', 'version': 1, 'included': 'not a list'}}) + assert layer.get(FEATURES, 'flag1') is not None + + +def test_layer_replacement_semantics(): + layer = OverrideLayer() + assert layer.is_empty is True + assert layer.get(FEATURES, 'flag1') is None + + layer.set_all({'flag1': flag('flag1')}, {}) + assert layer.is_empty is False + assert layer.get(FEATURES, 'flag1') is not None + + # A replacement is a full snapshot: entries absent from it are removed. + layer.set_all({'flag2': flag('flag2')}, {}) + assert layer.get(FEATURES, 'flag1') is None + assert layer.get(FEATURES, 'flag2') is not None + + layer.set_all({}, {}) + assert layer.is_empty is True + assert layer.get(FEATURES, 'flag2') is None + + +def test_layer_set_all_returns_previous_and_current_contents(): + layer = OverrideLayer() + previous, current = layer.set_all({'flag1': flag('flag1')}, {'seg': segment('seg')}) + assert previous == {FEATURES: {}, SEGMENTS: {}} + assert list(current[FEATURES].keys()) == ['flag1'] + assert list(current[SEGMENTS].keys()) == ['seg'] + previous, current = layer.set_all({}, {}) + assert list(previous[FEATURES].keys()) == ['flag1'] + assert current == {FEATURES: {}, SEGMENTS: {}} + + +def test_layer_all_returns_entries_of_a_kind(): + layer = OverrideLayer() + layer.set_all({'a': flag('a'), 'b': flag('b')}, {'s': segment('s')}) + assert sorted(layer.all(FEATURES).keys()) == ['a', 'b'] + assert list(layer.all(SEGMENTS).keys()) == ['s'] + assert all(item.is_override for item in layer.all(FEATURES).values()) + + +# --------------------------------------------------------------------------- +# Overlay +# --------------------------------------------------------------------------- + +def test_overlay_get_precedence(): + base = FakeBaseStore(flags={'both': flag('both', 1), 'base-only': flag('base-only', 1)}) + layer = OverrideLayer() + layer.set_all({'both': flag('both', 99), 'override-only': flag('override-only', 1)}, {}) + overlay = OverrideStoreView(base, layer) + + item = overlay.get(FEATURES, 'both') + assert item.version == 99 + assert item.is_override is True + + item = overlay.get(FEATURES, 'base-only') + assert item.is_override is False + + item = overlay.get(FEATURES, 'override-only') + assert item.is_override is True + + assert overlay.get(FEATURES, 'nowhere') is None + assert overlay.get(FEATURES, 'both', lambda x: x.version) == 99 + + +def test_overlay_get_serves_overrides_from_uninitialized_base(): + base = FakeBaseStore(initialized=False) + layer = OverrideLayer() + layer.set_all({'flag1': flag('flag1')}, {}) + overlay = OverrideStoreView(base, layer) + item = overlay.get(FEATURES, 'flag1') + assert item.is_override is True + assert overlay.initialized is False + + +def test_overlay_get_all_union(): + memory = InMemoryFeatureStore() + memory.set_basis({ + FEATURES: { + 'both': flag('both', 1).to_json_dict(), + 'base-only': flag('base-only', 1).to_json_dict(), + 'tombstone': {'key': 'tombstone', 'version': 5, 'deleted': True}, + }, + SEGMENTS: {}, + }) + layer = OverrideLayer() + layer.set_all({'both': flag('both', 99), 'tombstone': flag('tombstone', 1), 'override-only': flag('override-only', 1)}, {}) + overlay = OverrideStoreView(memory, layer) + + items = overlay.all(FEATURES) + assert sorted(items.keys()) == ['base-only', 'both', 'override-only', 'tombstone'] + assert items['both'].version == 99 + assert items['both'].is_override is True + assert items['base-only'].is_override is False + assert items['tombstone'].is_override is True, "an override wins over a deleted item" + assert items['override-only'].is_override is True + assert overlay.all(FEATURES, lambda x: len(x)) == 4 + + +def test_overlay_get_all_with_empty_layer_is_passthrough(): + base = FakeBaseStore(flags={'flag1': flag('flag1')}) + overlay = OverrideStoreView(base, OverrideLayer()) + assert list(overlay.all(FEATURES).keys()) == ['flag1'] + base.fail_all = True + with pytest.raises(RuntimeError): + overlay.all(FEATURES) + + +def test_overlay_get_all_serves_overrides_when_base_fails(): + base = FakeBaseStore() + base.fail_all = True + layer = OverrideLayer() + layer.set_all({'override-1': flag('override-1', 1), 'override-2': flag('override-2', 2)}, {}) + overlay = OverrideStoreView(base, layer) + items = overlay.all(FEATURES) + assert sorted(items.keys()) == ['override-1', 'override-2'] + assert all(item.is_override for item in items.values()) + + +def test_overlay_initialized_follows_the_base(): + base = FakeBaseStore(initialized=True) + overlay = OverrideStoreView(base, OverrideLayer()) + assert overlay.initialized is True + base._initialized = False + assert overlay.initialized is False + + +@pytest.mark.asyncio +async def test_async_overlay_get_and_all(): + base = FakeBaseStore(flags={'both': flag('both', 1), 'base-only': flag('base-only', 1)}, initialized=False) + layer = OverrideLayer() + layer.set_all({'both': flag('both', 99), 'override-only': flag('override-only', 1)}, {'seg': segment('seg')}) + overlay = AsyncOverrideStoreView(FakeAsyncBaseStore(base), layer) + + both = await overlay.get(FEATURES, 'both') + assert both.version == 99 and both.is_override is True + base_only = await overlay.get(FEATURES, 'base-only') + assert base_only.is_override is False + assert (await overlay.get(FEATURES, 'nowhere')) is None + assert (await overlay.get(SEGMENTS, 'seg')).is_override is True + + items = await overlay.all(FEATURES) + assert sorted(items.keys()) == ['base-only', 'both', 'override-only'] + assert items['both'].version == 99 + + +@pytest.mark.asyncio +async def test_async_overlay_all_when_base_fails(): + base = FakeBaseStore() + base.fail_all = True + layer = OverrideLayer() + overlay = AsyncOverrideStoreView(FakeAsyncBaseStore(base), layer) + with pytest.raises(RuntimeError): + await overlay.all(FEATURES) + layer.set_all({'override-1': flag('override-1')}, {}) + items = await overlay.all(FEATURES) + assert list(items.keys()) == ['override-1'] + + +# --------------------------------------------------------------------------- +# Sink +# --------------------------------------------------------------------------- + +class SinkFixture: + def __init__(self, base: FakeBaseStore): + self.base = base + self.layer = OverrideLayer() + self.notified: List[str] = [] + self.listening = True + self.sink = OverrideSinkImpl(self.layer, base, lambda key: self.notified.append(key), lambda: self.listening) + + def take_notified(self) -> List[str]: + result = sorted(self.notified) + del self.notified[:] + return result + + +def test_sink_notifies_on_add_change_remove(): + base = FakeBaseStore(flags={'flag1': flag('flag1', 1)}) + f = SinkFixture(base) + + # Adding an override is a change even though flag1 also exists in base data. + f.sink.set_overrides({'flag1': flag('flag1', 1), 'flag2': flag('flag2', 1)}, {}) + assert f.take_notified() == ['flag1', 'flag2'] + + # An identical replacement (rebuilt from scratch, new objects) changes nothing. + f.sink.set_overrides({'flag1': flag('flag1', 1), 'flag2': flag('flag2', 1)}, {}) + assert f.take_notified() == [] + + # Changing one entry notifies only that entry. + f.sink.set_overrides({'flag1': flag('flag1', 1), 'flag2': flag('flag2', 2)}, {}) + assert f.take_notified() == ['flag2'] + + # A change in content at the same version is a change. + f.sink.set_overrides({'flag1': flag('flag1', 1), 'flag2': FlagBuilder('flag2').version(2).on(False).build()}, {}) + assert f.take_notified() == ['flag2'] + + # Removing overrides notifies them: flag1 reverts to base data, flag2 to not-found. + f.sink.set_overrides({}, {}) + assert f.take_notified() == ['flag1', 'flag2'] + + +def test_sink_segment_override_fans_out_to_dependent_flags(): + dependent = FlagBuilder('dependent').version(1).on(True).variations(True, False).fallthrough_variation(1).rules( + FlagRuleBuilder().id('r').variation(0).clauses(make_clause_matching_segment_key('segment1')).build() + ).build() + base = FakeBaseStore( + flags={'dependent': dependent, 'unrelated': flag('unrelated')}, + segments={'segment1': segment('segment1', 1)}, + ) + f = SinkFixture(base) + f.sink.set_overrides({}, {'segment1': segment('segment1', 99)}) + # The segment itself is not a flag, so only the dependent flag is notified. + assert f.take_notified() == ['dependent'] + + +def test_sink_prerequisite_override_fans_out_through_the_chain(): + top = FlagBuilder('top').version(1).on(True).variations(True, False).fallthrough_variation(0).prerequisite('mid', 0).build() + mid = FlagBuilder('mid').version(1).on(True).variations(True, False).fallthrough_variation(0).prerequisite('leaf', 0).build() + base = FakeBaseStore(flags={'top': top, 'mid': mid, 'leaf': flag('leaf'), 'other': flag('other')}) + f = SinkFixture(base) + f.sink.set_overrides({'leaf': flag('leaf', 5)}, {}) + assert f.take_notified() == ['leaf', 'mid', 'top'] + + +def test_sink_removing_a_segment_override_notifies_flags_that_reference_it(): + # The override of "dependent" references segment "s". Removing the override of "s" alone + # changes what "dependent" evaluates to, so "dependent" is notified through the dependency + # fan-out even though its own override did not change. + dependent_override = FlagBuilder('dependent').version(2).on(True).variations(True, False).fallthrough_variation(1).rules( + FlagRuleBuilder().variation(0).clauses(make_clause_matching_segment_key('s')).build() + ).build() + base = FakeBaseStore(flags={'dependent': flag('dependent')}, segments={'s': segment('s')}) + f = SinkFixture(base) + f.sink.set_overrides({'dependent': dependent_override}, {'s': segment('s', 2)}) + f.take_notified() + f.sink.set_overrides({'dependent': dependent_override}, {}) + assert f.take_notified() == ['dependent'] + + +def test_sink_skips_change_computation_when_nothing_listens(): + base = FakeBaseStore(flags={'flag1': flag('flag1')}) + f = SinkFixture(base) + f.listening = False + f.sink.set_overrides({'flag1': flag('flag1', 2)}, {}) + assert f.take_notified() == [] + assert f.layer.get(FEATURES, 'flag1').version == 2 + + +def test_sink_notifies_directly_changed_keys_when_base_read_fails(): + base = FakeBaseStore(flags={'flag1': flag('flag1')}) + base.fail_all = True + f = SinkFixture(base) + f.sink.set_overrides({'flag1': flag('flag1', 2)}, {}) + assert f.take_notified() == ['flag1'] + + +def test_sink_invalid_snapshot_raises_and_changes_nothing(): + base = FakeBaseStore() + f = SinkFixture(base) + f.sink.set_overrides({'flag1': flag('flag1')}, {}) + f.take_notified() + with pytest.raises(ValueError): + f.sink.set_overrides({'flag1': {'key': 'flag1', 'version': 'bad'}}, {}) + assert f.layer.get(FEATURES, 'flag1').version == 1 + assert f.take_notified() == [] diff --git a/ldclient/testing/mock_components.py b/ldclient/testing/mock_components.py index ad93b32b..59c257e1 100644 --- a/ldclient/testing/mock_components.py +++ b/ldclient/testing/mock_components.py @@ -1,9 +1,16 @@ +import threading import time -from typing import Callable +from typing import Any, Callable, Dict, Optional +from ldclient.impl.util import _Success from ldclient.interfaces import ( + Basis, BigSegmentStore, BigSegmentStoreMetadata, + ChangeSetBuilder, + IntentCode, + ObjectKind, + OverrideSink, Selector ) @@ -54,3 +61,120 @@ def __init__(self, selector: Selector): def selector(self) -> Selector: return self._selector + + +class MockOverrideSource: + """ + An override source for tests. It pushes its current contents to the sink when started and + on every later call to set_overrides, and records its lifecycle calls. + """ + + def __init__(self, flags: Optional[Dict[str, Any]] = None, segments: Optional[Dict[str, Any]] = None): + self._flags: Dict[str, Any] = dict(flags or {}) + self._segments: Dict[str, Any] = dict(segments or {}) + self._sink: Optional[OverrideSink] = None + self.start_count = 0 + self.close_count = 0 + + def start(self, sink: OverrideSink) -> None: + self.start_count += 1 + self._sink = sink + sink.set_overrides(self._flags, self._segments) + + def close(self) -> None: + self.close_count += 1 + + def set_overrides(self, flags: Optional[Dict[str, Any]] = None, segments: Optional[Dict[str, Any]] = None) -> None: + """Replaces the source's contents and, once started, pushes them to the sink.""" + self._flags = dict(flags or {}) + self._segments = dict(segments or {}) + if self._sink is not None: + self._sink.set_overrides(self._flags, self._segments) + + @property + def builder(self) -> 'MockOverrideSourceBuilder': + return MockOverrideSourceBuilder(self) + + +class MockOverrideSourceBuilder: + """Wraps a MockOverrideSource so it can be passed to the data system configuration.""" + + def __init__(self, source: MockOverrideSource): + self._source = source + self.build_count = 0 + + def build(self, config) -> MockOverrideSource: + self.build_count += 1 + return self._source + + +class FailingOverrideSourceBuilder: + """A builder whose build raises, to simulate invalid override source configuration.""" + + def build(self, config): + raise ValueError("invalid override source configuration") + + +class HangingSynchronizer: + """ + A synchronizer that connects but never yields data, so the client stays uninitialized + until it is stopped. + """ + + def __init__(self): + self._stop = threading.Event() + + @property + def name(self) -> str: + return "HangingSynchronizer" + + def sync(self, ss): + self._stop.wait() + yield from () + + def stop(self): + self._stop.set() + + @property + def builder(self) -> 'MockDataSourceBuilder': + return MockDataSourceBuilder(self) + + +class StaticInitializer: + """ + An initializer that supplies fixed flag and segment definitions as a full-transfer basis + with a defined selector, so the client reports full data availability once it has applied + the data. + """ + + def __init__(self, flags: Optional[Dict[str, dict]] = None, segments: Optional[Dict[str, dict]] = None): + self._flags = dict(flags or {}) + self._segments = dict(segments or {}) + + @property + def name(self) -> str: + return "StaticInitializer" + + def fetch(self, ss): + builder = ChangeSetBuilder() + builder.start(IntentCode.TRANSFER_FULL) + for key, flag in self._flags.items(): + builder.add_put(ObjectKind.FLAG, key, flag.get('version', 1), flag) + for key, segment in self._segments.items(): + builder.add_put(ObjectKind.SEGMENT, key, segment.get('version', 1), segment) + change_set = builder.finish(Selector.new_selector('test-state', 1)) + return _Success(Basis(change_set=change_set, persist=False, environment_id=None)) + + @property + def builder(self) -> 'MockDataSourceBuilder': + return MockDataSourceBuilder(self) + + +class MockDataSourceBuilder: + """Wraps a ready-made data source so it can be passed to the data system configuration.""" + + def __init__(self, source): + self._source = source + + def build(self, config): + return self._source diff --git a/ldclient/testing/test_async_client_overrides.py b/ldclient/testing/test_async_client_overrides.py new file mode 100644 index 00000000..7373e1db --- /dev/null +++ b/ldclient/testing/test_async_client_overrides.py @@ -0,0 +1,193 @@ +""" +Tests for flag overrides through the async client. These mirror the key scenarios of the sync +client tests. +""" +import asyncio +import threading +from typing import Any, Dict, Optional + +import pytest + +from ldclient.async_client import AsyncLDClient +from ldclient.async_config import AsyncConfig, AsyncDataSystemConfig +from ldclient.context import Context +from ldclient.impl.aio.concurrency import AsyncEvent +from ldclient.impl.integrations.files.filedata import make_flag_with_value +from ldclient.testing.builders import FlagBuilder +from ldclient.testing.mock_async_components import MockAsyncEventProcessor +from ldclient.testing.mock_components import ( + FailingOverrideSourceBuilder, + MockDataSourceBuilder, + MockOverrideSource, + StaticInitializer +) + +user = Context.create('user-key') + + +class AsyncHangingSynchronizer: + """An async synchronizer that connects but never yields data.""" + + def __init__(self): + self._stop = AsyncEvent() + + @property + def name(self) -> str: + return "AsyncHangingSynchronizer" + + async def sync(self, ss): + await self._stop.wait() + return + yield + + async def stop(self) -> None: + self._stop.set() + + +class AsyncStaticInitializer(StaticInitializer): + async def fetch(self, ss): # type: ignore[override] + return super().fetch(ss) + + +def single_value_flag(key: str, value: Any) -> dict: + return make_flag_with_value(key, value).to_json_dict() + + +async def make_uninitialized_client(source: MockOverrideSource) -> AsyncLDClient: + datasystem = AsyncDataSystemConfig(synchronizers=[MockDataSourceBuilder(AsyncHangingSynchronizer())], override_source=source.builder) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor()) + client = AsyncLDClient(config) + await client.start(start_wait=0) + return client + + +async def make_initialized_client(flags: Dict[str, dict], source: MockOverrideSource, segments: Optional[Dict[str, dict]] = None) -> AsyncLDClient: + initializer = AsyncStaticInitializer(flags, segments or {}) + datasystem = AsyncDataSystemConfig(initializers=[MockDataSourceBuilder(initializer)], override_source=source.builder) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor()) + client = AsyncLDClient(config) + await client.start(start_wait=5) + assert await client.is_initialized() is True + return client + + +@pytest.mark.asyncio +async def test_override_is_served_when_client_is_not_initialized(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source) + try: + assert await client.is_initialized() is False + detail = await client.variation_detail('overridden-flag', user, False) + assert detail.value is True + assert detail.reason == {'kind': 'OFF', 'overrideAffected': True} + finally: + await client.close() + assert source.close_count == 1 + + +@pytest.mark.asyncio +async def test_non_overridden_flag_still_short_circuits_when_client_is_not_initialized(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source) + try: + detail = await client.variation_detail('other-flag', user, False) + assert detail.value is False + assert detail.reason == {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_override_removal_restores_short_circuit(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source) + try: + assert await client.variation('overridden-flag', user, False) is True + source.set_overrides({}, {}) + detail = await client.variation_detail('overridden-flag', user, False) + assert detail.reason == {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_invalid_override_source_configuration_fails_construction(): + datasystem = AsyncDataSystemConfig(synchronizers=[MockDataSourceBuilder(AsyncHangingSynchronizer())], override_source=FailingOverrideSourceBuilder()) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor()) + with pytest.raises(ValueError): + AsyncLDClient(config) + + +@pytest.mark.asyncio +async def test_override_takes_precedence_over_launchdarkly_data_and_all_flags_reflects_it(): + ld_flag = FlagBuilder('flag-precedence').version(100).on(False).off_variation(0).variations('ld-value').build().to_json_dict() + normal = FlagBuilder('flag-normal').version(100).on(False).off_variation(0).variations('normal-value').build().to_json_dict() + source = MockOverrideSource(flags={'flag-precedence': single_value_flag('flag-precedence', 'override-value')}) + client = await make_initialized_client({'flag-precedence': ld_flag, 'flag-normal': normal}, source) + try: + detail = await client.variation_detail('flag-precedence', user, 'default') + assert detail.value == 'override-value' + assert detail.reason == {'kind': 'OFF', 'overrideAffected': True} + detail = await client.variation_detail('flag-normal', user, 'default') + assert detail.reason == {'kind': 'OFF'} + + state = await client.all_flags_state(user, with_reasons=True) + assert state.to_values_map() == {'flag-precedence': 'override-value', 'flag-normal': 'normal-value'} + assert state.get_flag_reason('flag-precedence') == {'kind': 'OFF', 'overrideAffected': True} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_all_flags_state_contains_only_overrides_when_client_is_not_initialized(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source) + try: + state = await client.all_flags_state(user) + assert state.valid is True + assert state.to_values_map() == {'overridden-flag': True} + source.set_overrides({}, {}) + assert (await client.all_flags_state(user)).valid is False + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_flag_tracker_is_notified_of_override_changes_on_the_event_loop(): + source = MockOverrideSource() + client = await make_uninitialized_client(source) + try: + changes: asyncio.Queue = asyncio.Queue() + loop_thread = threading.current_thread() + listener_threads = [] + + def listener(change): + listener_threads.append(threading.current_thread()) + changes.put_nowait(change) + + client.flag_tracker.add_listener(listener) + + # The source pushes from a worker thread, as the file source does. The notification + # is delivered on the event loop's thread. + await asyncio.to_thread(source.set_overrides, {'overridden-flag': single_value_flag('overridden-flag', True)}, {}) + change = await asyncio.wait_for(changes.get(), 5) + assert change.key == 'overridden-flag' + assert listener_threads[0] is loop_thread + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_flag_value_change_listener_sees_override_value_changes(): + ld_flag = FlagBuilder('flag').version(100).on(False).off_variation(0).variations('ld-value').build().to_json_dict() + source = MockOverrideSource() + client = await make_initialized_client({'flag': ld_flag}, source) + try: + changes: asyncio.Queue = asyncio.Queue() + await client.flag_tracker.add_flag_value_change_listener('flag', user, changes.put_nowait) + await asyncio.to_thread(source.set_overrides, {'flag': single_value_flag('flag', 'override-value')}, {}) + change = await asyncio.wait_for(changes.get(), 5) + assert change.old_value == 'ld-value' + assert change.new_value == 'override-value' + finally: + await client.close() diff --git a/ldclient/testing/test_ldclient_override_vectors.py b/ldclient/testing/test_ldclient_override_vectors.py new file mode 100644 index 00000000..d9533c3d --- /dev/null +++ b/ldclient/testing/test_ldclient_override_vectors.py @@ -0,0 +1,85 @@ +""" +Runs the OVERRIDE specification test vectors. Each vector sets up LaunchDarkly data, an +override layer, and an initialization state. The test evaluates one flag through the full +client stack and checks the value, the variation index, and the reason. +""" +import json +import os +from typing import Any, Dict + +import pytest + +from ldclient.client import Config, Context, LDClient +from ldclient.datasystem import custom +from ldclient.impl.integrations.files.filedata import make_flag_with_value +from ldclient.testing.mock_components import ( + HangingSynchronizer, + MockOverrideSource, + StaticInitializer +) +from ldclient.testing.stub_util import MockEventProcessor + +VECTORS_PATH = os.path.join(os.path.dirname(__file__), 'testdata', 'override-vectors', 'vectors.json') + +# The vectors' semantics are versioned. A schema change means this runner needs review. +SUPPORTED_SCHEMA_VERSION = '0.4.0' + + +def load_vectors(): + with open(VECTORS_PATH, 'r') as f: + document = json.load(f) + assert document['schemaVersion'] == SUPPORTED_SCHEMA_VERSION, "the vectors changed schema; review this runner against the new schema before updating" + assert len(document['vectors']) > 0 + return document['vectors'] + + +def vector_id(vector: Dict[str, Any]) -> str: + return "%s: %s" % (vector['group'], vector['description']) + + +def override_flags(overrides: Dict[str, Any]) -> Dict[str, Any]: + flags = dict(overrides.get('flags', {})) + for key, value in overrides.get('flagValues', {}).items(): + flags[key] = make_flag_with_value(key, value) + return flags + + +def make_client(vector: Dict[str, Any]) -> LDClient: + source = MockOverrideSource(flags=override_flags(vector['overrides']), segments=dict(vector['overrides'].get('segments', {}))) + ld_data = vector['launchDarklyData'] + datasystem = custom().overrides(source.builder) + if ld_data['initialized']: + datasystem.initializers([StaticInitializer(ld_data.get('flags', {}), ld_data.get('segments', {})).builder]) + start_wait = 5 + else: + # With no sources at all, the client would consider cached data available rather than + # applying its not-initialized handling. A synchronizer that never delivers avoids that. + datasystem.synchronizers(HangingSynchronizer().builder) + start_wait = 0 + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem.build(), event_processor_class=MockEventProcessor) + return LDClient(config, start_wait=start_wait) + + +def assert_reason(expected: Dict[str, Any], actual: Dict[str, Any]) -> None: + """ + The reason is compared only on the fields present in the expected reason. The + override-affected indicator is a tri-state collapse: an expected reason that omits it + requires the actual reason to omit it or report it as false. + """ + for name, value in expected.items(): + assert actual.get(name) == value, "reason property %s: expected %r in %r" % (name, value, actual) + if 'overrideAffected' not in expected: + assert actual.get('overrideAffected', False) is False, "reason must not be override-affected: %r" % actual + + +@pytest.mark.parametrize("vector", load_vectors(), ids=vector_id) +def test_override_spec_vector(vector: Dict[str, Any]): + with make_client(vector) as client: + assert client.is_initialized() is vector['launchDarklyData']['initialized'] + evaluate = vector['evaluate'] + detail = client.variation_detail(evaluate['flagKey'], Context.from_dict(evaluate['context']), evaluate['defaultValue']) + + expect = vector['expect'] + assert detail.value == expect['value'], "value" + assert detail.variation_index == expect['variationIndex'], "variationIndex" + assert_reason(expect['reason'], detail.reason) diff --git a/ldclient/testing/test_ldclient_overrides.py b/ldclient/testing/test_ldclient_overrides.py new file mode 100644 index 00000000..030a603f --- /dev/null +++ b/ldclient/testing/test_ldclient_overrides.py @@ -0,0 +1,255 @@ +""" +Tests for flag overrides through the client: the override source lifecycle, the overlay at the +store read boundary, the not-initialized gate, the all-flags state, and flag change notifications. +""" +import logging +from queue import Empty, Queue +from typing import Any, Dict, Optional + +import pytest + +from ldclient.client import Config, Context, LDClient +from ldclient.datasystem import custom +from ldclient.evaluation import EvaluationDetail +from ldclient.impl.datasystem.fdv1 import FDv1 +from ldclient.impl.integrations.files.filedata import make_flag_with_value +from ldclient.interfaces import DataSourceState, FlagChange +from ldclient.testing.builders import ( + FlagBuilder, + FlagRuleBuilder, + SegmentBuilder, + make_clause_matching_segment_key +) +from ldclient.testing.mock_components import ( + FailingOverrideSourceBuilder, + HangingSynchronizer, + MockOverrideSource, + StaticInitializer +) +from ldclient.testing.stub_util import MockEventProcessor + +user = Context.create('user-key') + + +def single_value_flag(key: str, value: Any) -> dict: + return make_flag_with_value(key, value).to_json_dict() + + +def make_uninitialized_client(source: MockOverrideSource) -> LDClient: + """A client whose data system can never obtain LaunchDarkly data, with the given override source.""" + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + return LDClient(config, start_wait=0) + + +def make_initialized_client(flags: Dict[str, dict], source: MockOverrideSource, segments: Optional[Dict[str, dict]] = None) -> LDClient: + """A client that has initialized with the given LaunchDarkly data, with the given override source.""" + initializer = StaticInitializer(flags, segments or {}) + datasystem = custom().initializers([initializer.builder]).overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + client = LDClient(config, start_wait=5) + assert client.is_initialized() is True + return client + + +def warnings_containing(caplog, text: str): + return [r for r in caplog.records if r.levelno == logging.WARNING and text in r.getMessage()] + + +def test_override_is_served_when_client_is_not_initialized(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + assert client.is_initialized() is False + detail = client.variation_detail('overridden-flag', user, False) + assert detail.value is True + assert detail.variation_index == 0 + assert detail.reason == {'kind': 'OFF', 'overrideAffected': True} + + +def test_non_overridden_flag_still_short_circuits_when_client_is_not_initialized(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + detail = client.variation_detail('other-flag', user, False) + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'}) + + +def test_override_removal_restores_short_circuit(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + assert client.variation('overridden-flag', user, False) is True + source.set_overrides({}, {}) + detail = client.variation_detail('overridden-flag', user, False) + assert detail.value is False + assert detail.reason == {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'} + + +def test_override_source_is_started_before_the_constructor_returns(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = make_uninitialized_client(source) + try: + assert source.start_count == 1 + assert client.variation('overridden-flag', user, False) is True + finally: + client.close() + + +def test_override_source_is_closed_when_the_client_is_closed(): + source = MockOverrideSource() + client = make_uninitialized_client(source) + assert source.close_count == 0 + client.close() + assert source.close_count == 1 + + +def test_invalid_override_source_configuration_fails_construction(): + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(FailingOverrideSourceBuilder()).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + with pytest.raises(ValueError): + LDClient(config, start_wait=0) + + +def test_override_source_is_not_started_when_offline(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + datasystem = custom().overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, offline=True) + with LDClient(config, start_wait=0) as client: + assert source.start_count == 0 + assert client.variation('overridden-flag', user, False) is False + + +def test_data_system_without_override_source_reports_none_configured(): + datasystem = custom().synchronizers(HangingSynchronizer().builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + with LDClient(config, start_wait=0) as client: + assert client._data_system.override_source_configured is False + detail = client.variation_detail('any-flag', user, False) + assert detail.reason == {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'} + + +def test_legacy_data_system_reports_no_override_source(): + config = Config(sdk_key='SDK_KEY', offline=True) + with LDClient(config) as client: + assert isinstance(client._data_system, FDv1) + assert client._data_system.override_source_configured is False + + +def test_override_takes_precedence_over_launchdarkly_data(): + ld_flag = FlagBuilder('flag-precedence').version(100).on(False).off_variation(0).variations('ld-value').build().to_json_dict() + normal = FlagBuilder('flag-normal').version(100).on(False).off_variation(0).variations('normal-value').build().to_json_dict() + source = MockOverrideSource(flags={'flag-precedence': single_value_flag('flag-precedence', 'override-value')}) + with make_initialized_client({'flag-precedence': ld_flag, 'flag-normal': normal}, source) as client: + detail = client.variation_detail('flag-precedence', user, 'default') + assert detail.value == 'override-value' + assert detail.reason == {'kind': 'OFF', 'overrideAffected': True} + detail = client.variation_detail('flag-normal', user, 'default') + assert detail.value == 'normal-value' + assert detail.reason == {'kind': 'OFF'} + + +def test_full_flag_override_evaluates_targeting_rules_and_referenced_override_segment(): + segment_flag = FlagBuilder('flag-segment').version(1).on(True).off_variation(0).fallthrough_variation(0).variations('not-included', 'included').rules( + FlagRuleBuilder().id('segment-rule').variation(1).clauses(make_clause_matching_segment_key('overridden-segment')).build() + ).build().to_json_dict() + ld_segment = SegmentBuilder('overridden-segment').version(100).build().to_json_dict() + override_segment = SegmentBuilder('overridden-segment').version(101).included(user.key).build().to_json_dict() + source = MockOverrideSource(flags={'flag-segment': segment_flag}, segments={'overridden-segment': override_segment}) + with make_initialized_client({}, source, segments={'overridden-segment': ld_segment}) as client: + detail = client.variation_detail('flag-segment', user, 'default') + assert detail.value == 'included' + assert detail.reason == {'kind': 'RULE_MATCH', 'ruleIndex': 0, 'ruleId': 'segment-rule', 'overrideAffected': True} + + +def test_overridden_prerequisite_marks_the_dependent_flag(): + dependent = FlagBuilder('dependent').version(1).on(True).off_variation(0).fallthrough_variation(1).variations('prereq-failed', 'affected-value').prerequisite('prereq', 1).build().to_json_dict() + ld_prereq = FlagBuilder('prereq').version(100).on(False).off_variation(0).variations('a', 'b').build().to_json_dict() + override_prereq = FlagBuilder('prereq').version(200).on(True).off_variation(0).fallthrough_variation(1).variations('a', 'b').build().to_json_dict() + source = MockOverrideSource(flags={'prereq': override_prereq}) + with make_initialized_client({'dependent': dependent, 'prereq': ld_prereq}, source) as client: + detail = client.variation_detail('dependent', user, 'default') + assert detail.value == 'affected-value' + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + + +def test_all_flags_state_contains_only_overrides_when_client_is_not_initialized(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + state = client.all_flags_state(user) + assert state.valid is True + assert state.to_values_map() == {'overridden-flag': True} + + +def test_all_flags_state_overrides_only_warning_is_logged_once(caplog): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + with caplog.at_level(logging.WARNING): + assert client.all_flags_state(user).valid is True + assert client.all_flags_state(user).valid is True + assert len(warnings_containing(caplog, 'Returning only flags from the override layer')) == 1 + + +def test_all_flags_state_is_invalid_when_not_initialized_and_override_layer_is_empty(): + source = MockOverrideSource() + with make_uninitialized_client(source) as client: + state = client.all_flags_state(user) + assert state.valid is False + assert state.to_values_map() == {} + + +def test_all_flags_state_reflects_overrides_when_initialized(): + ld_flag = FlagBuilder('flag-precedence').version(100).on(False).off_variation(0).variations('ld-value').build().to_json_dict() + normal = FlagBuilder('flag-normal').version(100).on(False).off_variation(0).variations('normal-value').build().to_json_dict() + source = MockOverrideSource(flags={ + 'flag-precedence': single_value_flag('flag-precedence', 'override-value'), + 'override-only': single_value_flag('override-only', 'only-value'), + }) + with make_initialized_client({'flag-precedence': ld_flag, 'flag-normal': normal}, source) as client: + state = client.all_flags_state(user, with_reasons=True) + assert state.valid is True + assert state.to_values_map() == {'flag-precedence': 'override-value', 'flag-normal': 'normal-value', 'override-only': 'only-value'} + assert state.get_flag_reason('flag-precedence') == {'kind': 'OFF', 'overrideAffected': True} + assert state.get_flag_reason('flag-normal') == {'kind': 'OFF'} + + +def test_flag_tracker_is_notified_of_override_changes(): + source = MockOverrideSource() + with make_uninitialized_client(source) as client: + changes: Queue = Queue() + client.flag_tracker.add_listener(lambda change: changes.put(change)) + + source.set_overrides({'overridden-flag': single_value_flag('overridden-flag', True)}, {}) + change = changes.get(timeout=5) + assert isinstance(change, FlagChange) + assert change.key == 'overridden-flag' + + # An identical snapshot notifies nothing. + source.set_overrides({'overridden-flag': single_value_flag('overridden-flag', True)}, {}) + with pytest.raises(Empty): + changes.get(timeout=0.3) + + source.set_overrides({}, {}) + assert changes.get(timeout=5).key == 'overridden-flag' + + +def test_flag_value_change_listener_sees_override_value_changes(): + ld_flag = FlagBuilder('flag').version(100).on(False).off_variation(0).variations('ld-value').build().to_json_dict() + source = MockOverrideSource() + with make_initialized_client({'flag': ld_flag}, source) as client: + changes: Queue = Queue() + client.flag_tracker.add_flag_value_change_listener('flag', user, lambda change: changes.put(change)) + + source.set_overrides({'flag': single_value_flag('flag', 'override-value')}, {}) + change = changes.get(timeout=5) + assert change.old_value == 'ld-value' + assert change.new_value == 'override-value' + + source.set_overrides({}, {}) + change = changes.get(timeout=5) + assert change.old_value == 'override-value' + assert change.new_value == 'ld-value' + + +def test_data_source_status_is_unaffected_by_overrides(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + assert client.is_initialized() is False + assert client.data_source_status_provider.status.state == DataSourceState.INITIALIZING diff --git a/ldclient/testing/testdata/override-vectors/vectors.json b/ldclient/testing/testdata/override-vectors/vectors.json new file mode 100644 index 00000000..48c3347c --- /dev/null +++ b/ldclient/testing/testdata/override-vectors/vectors.json @@ -0,0 +1,316 @@ +{ + "schemaVersion": "0.4.0", + "vectors": [ + { + "description": "An override takes precedence over LaunchDarkly data for the same flag key", + "group": "precedence", + "launchDarklyData": { + "initialized": true, + "flags": { + "flag-a": { + "key": "flag-a", + "version": 7, + "on": true, + "variations": ["ld-value"], + "fallthrough": { "variation": 0 }, + "offVariation": 0, + "salt": "salt-a" + } + }, + "segments": {} + }, + "overrides": { + "flagValues": { "flag-a": "override-value" } + }, + "evaluate": { + "flagKey": "flag-a", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "override-value", + "variationIndex": 0, + "reason": { "kind": "OFF", "overrideAffected": true }, + "summaryOverrideAffected": true + } + }, + { + "description": "An overridden flag is served when the SDK has not initialized from LaunchDarkly", + "group": "uninitialized", + "launchDarklyData": { + "initialized": false, + "flags": {}, + "segments": {} + }, + "overrides": { + "flagValues": { "flag-a": "override-value" } + }, + "evaluate": { + "flagKey": "flag-a", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "override-value", + "variationIndex": 0, + "reason": { "kind": "OFF", "overrideAffected": true }, + "summaryOverrideAffected": true + } + }, + { + "description": "A flag absent from the override layer is evaluated from LaunchDarkly data and is not marked", + "group": "passthrough", + "launchDarklyData": { + "initialized": true, + "flags": { + "flag-b": { + "key": "flag-b", + "version": 3, + "on": true, + "variations": ["off-value", "ld-value"], + "fallthrough": { "variation": 1 }, + "offVariation": 0, + "salt": "salt-b" + } + }, + "segments": {} + }, + "overrides": { + "flagValues": { "flag-a": "override-value" } + }, + "evaluate": { + "flagKey": "flag-b", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "ld-value", + "variationIndex": 1, + "reason": { "kind": "FALLTHROUGH" }, + "summaryOverrideAffected": false + } + }, + { + "description": "A flag absent from the override layer still short-circuits when the SDK has not initialized", + "group": "passthrough", + "launchDarklyData": { + "initialized": false, + "flags": {}, + "segments": {} + }, + "overrides": { + "flagValues": { "flag-a": "override-value" } + }, + "evaluate": { + "flagKey": "flag-b", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "fallback", + "variationIndex": null, + "reason": { "kind": "ERROR", "errorKind": "CLIENT_NOT_READY" } + } + }, + { + "description": "A full flag override with a matching targeting rule serves the rule's variation and is marked", + "group": "full-evaluation", + "launchDarklyData": { + "initialized": true, + "flags": {}, + "segments": {} + }, + "overrides": { + "flags": { + "flag-c": { + "key": "flag-c", + "version": 1, + "on": true, + "variations": ["default-value", "eu-value"], + "fallthrough": { "variation": 0 }, + "offVariation": 0, + "rules": [ + { + "id": "rule-eu", + "variation": 1, + "clauses": [ + { + "contextKind": "user", + "attribute": "region", + "op": "in", + "values": ["eu"], + "negate": false + } + ] + } + ], + "salt": "salt-c" + } + } + }, + "evaluate": { + "flagKey": "flag-c", + "context": { "kind": "user", "key": "user-1", "region": "eu" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "eu-value", + "variationIndex": 1, + "reason": { "kind": "RULE_MATCH", "ruleIndex": 0, "ruleId": "rule-eu", "overrideAffected": true }, + "summaryOverrideAffected": true + } + }, + { + "description": "An override flag resolves a referenced segment through the merged view while the SDK is uninitialized", + "group": "merged-view", + "launchDarklyData": { + "initialized": false, + "flags": {}, + "segments": {} + }, + "overrides": { + "flags": { + "flag-d": { + "key": "flag-d", + "version": 1, + "on": true, + "variations": ["default-value", "member-value"], + "fallthrough": { "variation": 0 }, + "offVariation": 0, + "rules": [ + { + "id": "rule-seg", + "variation": 1, + "clauses": [ + { + "attribute": "", + "op": "segmentMatch", + "values": ["seg-1"], + "negate": false + } + ] + } + ], + "salt": "salt-d" + } + }, + "segments": { + "seg-1": { + "key": "seg-1", + "version": 1, + "included": ["user-1"], + "excluded": [], + "salt": "seg-salt-1" + } + } + }, + "evaluate": { + "flagKey": "flag-d", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "member-value", + "variationIndex": 1, + "reason": { "kind": "RULE_MATCH", "ruleIndex": 0, "ruleId": "rule-seg", "overrideAffected": true }, + "summaryOverrideAffected": true + } + }, + { + "description": "A flag is marked overrideAffected when it matches via an overridden segment", + "group": "marking", + "launchDarklyData": { + "initialized": true, + "flags": { + "flag-f": { + "key": "flag-f", + "version": 4, + "on": true, + "variations": ["default-value", "member-value"], + "fallthrough": { "variation": 0 }, + "offVariation": 0, + "rules": [ + { + "id": "rule-seg", + "variation": 1, + "clauses": [ + { + "attribute": "", + "op": "segmentMatch", + "values": ["seg-2"], + "negate": false + } + ] + } + ], + "salt": "salt-f" + } + }, + "segments": { + "seg-2": { + "key": "seg-2", + "version": 1, + "included": [], + "excluded": [], + "salt": "seg-salt-2" + } + } + }, + "overrides": { + "segments": { + "seg-2": { + "key": "seg-2", + "version": 2, + "included": ["user-1"], + "excluded": [], + "salt": "seg-salt-2" + } + } + }, + "evaluate": { + "flagKey": "flag-f", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "member-value", + "variationIndex": 1, + "reason": { "kind": "RULE_MATCH", "ruleIndex": 0, "ruleId": "rule-seg", "overrideAffected": true }, + "summaryOverrideAffected": true + } + }, + { + "description": "A malformed override flag returns the caller default with an error reason and is marked overrideAffected", + "group": "error-handling", + "launchDarklyData": { + "initialized": true, + "flags": {}, + "segments": {} + }, + "overrides": { + "flags": { + "flag-e": { + "key": "flag-e", + "version": 1, + "on": true, + "variations": ["only-value"], + "fallthrough": { "variation": 5 }, + "offVariation": 0, + "salt": "salt-e" + } + } + }, + "evaluate": { + "flagKey": "flag-e", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "fallback", + "variationIndex": null, + "reason": { "kind": "ERROR", "errorKind": "MALFORMED_FLAG", "overrideAffected": true }, + "summaryOverrideAffected": true + } + } + ] +} From 60aa3330cded73e1671aa3dc2600ca85b02dc102 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:09:05 +0000 Subject: [PATCH 03/26] test: Expect the fallthrough reason for value-only overrides --- ldclient/testing/test_async_client_overrides.py | 6 +++--- ldclient/testing/test_ldclient_overrides.py | 6 +++--- ldclient/testing/testdata/override-vectors/vectors.json | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ldclient/testing/test_async_client_overrides.py b/ldclient/testing/test_async_client_overrides.py index 7373e1db..d8e53bfb 100644 --- a/ldclient/testing/test_async_client_overrides.py +++ b/ldclient/testing/test_async_client_overrides.py @@ -79,7 +79,7 @@ async def test_override_is_served_when_client_is_not_initialized(): assert await client.is_initialized() is False detail = await client.variation_detail('overridden-flag', user, False) assert detail.value is True - assert detail.reason == {'kind': 'OFF', 'overrideAffected': True} + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} finally: await client.close() assert source.close_count == 1 @@ -127,13 +127,13 @@ async def test_override_takes_precedence_over_launchdarkly_data_and_all_flags_re try: detail = await client.variation_detail('flag-precedence', user, 'default') assert detail.value == 'override-value' - assert detail.reason == {'kind': 'OFF', 'overrideAffected': True} + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} detail = await client.variation_detail('flag-normal', user, 'default') assert detail.reason == {'kind': 'OFF'} state = await client.all_flags_state(user, with_reasons=True) assert state.to_values_map() == {'flag-precedence': 'override-value', 'flag-normal': 'normal-value'} - assert state.get_flag_reason('flag-precedence') == {'kind': 'OFF', 'overrideAffected': True} + assert state.get_flag_reason('flag-precedence') == {'kind': 'FALLTHROUGH', 'overrideAffected': True} finally: await client.close() diff --git a/ldclient/testing/test_ldclient_overrides.py b/ldclient/testing/test_ldclient_overrides.py index 030a603f..f6b54a44 100644 --- a/ldclient/testing/test_ldclient_overrides.py +++ b/ldclient/testing/test_ldclient_overrides.py @@ -63,7 +63,7 @@ def test_override_is_served_when_client_is_not_initialized(): detail = client.variation_detail('overridden-flag', user, False) assert detail.value is True assert detail.variation_index == 0 - assert detail.reason == {'kind': 'OFF', 'overrideAffected': True} + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} def test_non_overridden_flag_still_short_circuits_when_client_is_not_initialized(): @@ -140,7 +140,7 @@ def test_override_takes_precedence_over_launchdarkly_data(): with make_initialized_client({'flag-precedence': ld_flag, 'flag-normal': normal}, source) as client: detail = client.variation_detail('flag-precedence', user, 'default') assert detail.value == 'override-value' - assert detail.reason == {'kind': 'OFF', 'overrideAffected': True} + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} detail = client.variation_detail('flag-normal', user, 'default') assert detail.value == 'normal-value' assert detail.reason == {'kind': 'OFF'} @@ -206,7 +206,7 @@ def test_all_flags_state_reflects_overrides_when_initialized(): state = client.all_flags_state(user, with_reasons=True) assert state.valid is True assert state.to_values_map() == {'flag-precedence': 'override-value', 'flag-normal': 'normal-value', 'override-only': 'only-value'} - assert state.get_flag_reason('flag-precedence') == {'kind': 'OFF', 'overrideAffected': True} + assert state.get_flag_reason('flag-precedence') == {'kind': 'FALLTHROUGH', 'overrideAffected': True} assert state.get_flag_reason('flag-normal') == {'kind': 'OFF'} diff --git a/ldclient/testing/testdata/override-vectors/vectors.json b/ldclient/testing/testdata/override-vectors/vectors.json index 48c3347c..82dc62da 100644 --- a/ldclient/testing/testdata/override-vectors/vectors.json +++ b/ldclient/testing/testdata/override-vectors/vectors.json @@ -30,7 +30,7 @@ "expect": { "value": "override-value", "variationIndex": 0, - "reason": { "kind": "OFF", "overrideAffected": true }, + "reason": { "kind": "FALLTHROUGH", "overrideAffected": true }, "summaryOverrideAffected": true } }, @@ -53,7 +53,7 @@ "expect": { "value": "override-value", "variationIndex": 0, - "reason": { "kind": "OFF", "overrideAffected": true }, + "reason": { "kind": "FALLTHROUGH", "overrideAffected": true }, "summaryOverrideAffected": true } }, From 5e4ea1721b214f7f16ac147b08a81ed2d8463cbb Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Sat, 3 Oct 2026 00:42:40 +0000 Subject: [PATCH 04/26] fix: Ask the override layer for the key before the client has LaunchDarkly data Before initialization the clients consult the override layer directly. A flag the layer does not hold gets the not-ready handling it had without an override source, for an invalid context, a store outage, and an uninitialized persistent store alike. The all-flags state is unavailable when the layer is empty. The sync client stops the components it started when the data system fails to start. --- ldclient/async_client.py | 49 +++---- ldclient/client.py | 66 +++++---- ldclient/impl/datasystem/__init__.py | 17 +-- ldclient/impl/datasystem/async_fdv1.py | 4 +- ldclient/impl/datasystem/async_fdv2.py | 4 +- ldclient/impl/datasystem/fdv1.py | 4 +- ldclient/impl/datasystem/fdv2.py | 4 +- ldclient/testing/mock_components.py | 8 +- .../testing/test_async_client_overrides.py | 127 ++++++++++++++++- ldclient/testing/test_ldclient_overrides.py | 128 ++++++++++++++++-- 10 files changed, 334 insertions(+), 77 deletions(-) diff --git a/ldclient/async_client.py b/ldclient/async_client.py index d04f2ab6..d5191eca 100644 --- a/ldclient/async_client.py +++ b/ldclient/async_client.py @@ -505,36 +505,37 @@ def not_ready() -> Tuple[EvaluationDetail, Optional[FeatureFlag]]: self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) return EvaluationDetail(default, None, reason), None - no_launchdarkly_data = False + flag: Any = None availability = await self._data_system.data_availability() if availability != DataAvailability.REFRESHED: if availability == DataAvailability.CACHED: if not self._eval_cached_data_warned: self._eval_cached_data_warned = True log.warning("Feature Flag evaluation attempted before client has initialized - using last known values from feature store for feature key: " + key + ". This message is logged once.") - elif self._data_system.override_source_configured: - # No data from LaunchDarkly is available. The store read below still finds an - # entry that the override layer holds, and the SDK serves it. A miss returns the - # not-ready default. - no_launchdarkly_data = True else: - return not_ready() + # No data from LaunchDarkly is available. A flag that the override layer holds + # is served anyway. A flag the layer does not hold gets the not-ready default, + # as it does without an override source. + layer = self._data_system.override_layer + if layer is not None: + flag = layer.get(FEATURES, key) + if flag is None: + return not_ready() if not context.valid: log.warning("Context was invalid for flag evaluation (%s); returning default value" % context.error) return EvaluationDetail(default, None, error_reason('USER_NOT_SPECIFIED')), None - try: - flag = await self._data_system.store.get(FEATURES, key) - except Exception as e: - log.error("Unexpected error while retrieving feature flag \"%s\": %s" % (key, repr(e))) - log.debug(traceback.format_exc()) - reason = error_reason('EXCEPTION') - self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) - return EvaluationDetail(default, None, reason), None + if flag is None: + try: + flag = await self._data_system.store.get(FEATURES, key) + except Exception as e: + log.error("Unexpected error while retrieving feature flag \"%s\": %s" % (key, repr(e))) + log.debug(traceback.format_exc()) + reason = error_reason('EXCEPTION') + self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) + return EvaluationDetail(default, None, reason), None if not flag: - if no_launchdarkly_data: - return not_ready() reason = error_reason('FLAG_NOT_FOUND') self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) return EvaluationDetail(default, None, reason), None @@ -591,13 +592,15 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState if not self._all_flags_cached_data_warned: self._all_flags_cached_data_warned = True log.warning("all_flags_state() called before client has finished initializing! Using last known values from feature store. This message is logged once.") - elif self._data_system.override_source_configured: - # No data from LaunchDarkly is available. The store read below returns only the - # entries that the override layer holds. The result decides the state. - overrides_only = True else: - log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state") - return FeatureFlagsState(False) + # No data from LaunchDarkly is available. When the override layer holds entries, + # the state is built from them alone. Otherwise the state is unavailable, as it + # is without an override source. + layer = self._data_system.override_layer + if layer is None or layer.is_empty: + log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state") + return FeatureFlagsState(False) + overrides_only = True if not context.valid: log.warning("Context was invalid for all_flags_state (%s); returning default value" % context.error) diff --git a/ldclient/client.py b/ldclient/client.py index f6067723..c03de402 100644 --- a/ldclient/client.py +++ b/ldclient/client.py @@ -212,7 +212,13 @@ def __start_up(self, start_wait: float): self.__register_plugins(environment_metadata) update_processor_ready = threading.Event() - self._data_system.start(update_processor_ready) + try: + self._data_system.start(update_processor_ready) + except BaseException: + # A data system that fails to start fails the constructor. The components that + # are already running stop here, so the failure leaves no threads behind. + self._close_components() + raise if not self._config.offline and not self._config.use_ldd: if start_wait > 60: @@ -282,6 +288,15 @@ def close(self): self._data_system.stop() self.__big_segment_store_manager.stop() + def _close_components(self): + """ + Stops the SDK components after a failed start-up. The event processor is last because + it may still be sending events the other components generated. + """ + self._data_system.stop() + self.__big_segment_store_manager.stop() + self._event_processor.stop() + # These magic methods allow a client object to be automatically cleaned up by the "with" scope operator def __enter__(self): return self @@ -474,7 +489,7 @@ def not_ready() -> Tuple[EvaluationDetail, Optional[FeatureFlag]]: self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) return EvaluationDetail(default, None, reason), None - no_launchdarkly_data = False + flag: Any = None availability = self._data_system.data_availability if availability != DataAvailability.REFRESHED: if availability == DataAvailability.CACHED: @@ -483,29 +498,30 @@ def not_ready() -> Tuple[EvaluationDetail, Optional[FeatureFlag]]: if not self._eval_cached_data_warned: self._eval_cached_data_warned = True log.warning("Feature Flag evaluation attempted before client has initialized - using last known values from feature store for feature key: " + key + ". This message is logged once.") - elif self._data_system.override_source_configured: - # No data from LaunchDarkly is available. The store read below still finds an - # entry that the override layer holds, and the SDK serves it. A miss returns the - # not-ready default. - no_launchdarkly_data = True else: - return not_ready() + # No data from LaunchDarkly is available. A flag that the override layer holds + # is served anyway. A flag the layer does not hold gets the not-ready default, + # as it does without an override source. + layer = self._data_system.override_layer + if layer is not None: + flag = layer.get(FEATURES, key) + if flag is None: + return not_ready() if not context.valid: log.warning("Context was invalid for flag evaluation (%s); returning default value" % context.error) return EvaluationDetail(default, None, error_reason('USER_NOT_SPECIFIED')), None - try: - flag = self._data_system.store.get(FEATURES, key) - except Exception as e: - log.error("Unexpected error while retrieving feature flag \"%s\": %s" % (key, repr(e))) - log.debug(traceback.format_exc()) - reason = error_reason('EXCEPTION') - self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) - return EvaluationDetail(default, None, reason), None + if flag is None: + try: + flag = self._data_system.store.get(FEATURES, key) + except Exception as e: + log.error("Unexpected error while retrieving feature flag \"%s\": %s" % (key, repr(e))) + log.debug(traceback.format_exc()) + reason = error_reason('EXCEPTION') + self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) + return EvaluationDetail(default, None, reason), None if not flag: - if no_launchdarkly_data: - return not_ready() reason = error_reason('FLAG_NOT_FOUND') self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) return EvaluationDetail(default, None, reason), None @@ -566,13 +582,15 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: if not self._all_flags_cached_data_warned: self._all_flags_cached_data_warned = True log.warning("all_flags_state() called before client has finished initializing! Using last known values from feature store. This message is logged once.") - elif self._data_system.override_source_configured: - # No data from LaunchDarkly is available. The store read below returns only the - # entries that the override layer holds. The result decides the state. - overrides_only = True else: - log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state") - return FeatureFlagsState(False) + # No data from LaunchDarkly is available. When the override layer holds entries, + # the state is built from them alone. Otherwise the state is unavailable, as it + # is without an override source. + layer = self._data_system.override_layer + if layer is None or layer.is_empty: + log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state") + return FeatureFlagsState(False) + overrides_only = True if not context.valid: log.warning("Context was invalid for all_flags_state (%s); returning default value" % context.error) diff --git a/ldclient/impl/datasystem/__init__.py b/ldclient/impl/datasystem/__init__.py index 88f1f7b7..6799351f 100644 --- a/ldclient/impl/datasystem/__init__.py +++ b/ldclient/impl/datasystem/__init__.py @@ -10,6 +10,7 @@ if TYPE_CHECKING: from ldclient.impl.aio.concurrency import AsyncEvent + from ldclient.impl.overrides.layer import OverrideLayer from ldclient.impl.listeners import Listeners from ldclient.interfaces import ( @@ -160,12 +161,12 @@ def environment_id(self) -> Optional[str]: @property @abstractmethod - def override_source_configured(self) -> bool: + def override_layer(self) -> Optional["OverrideLayer"]: """ - Reports whether the data system was built with an override source. The value is - fixed at construction. When true, the store served by :attr:`store` overlays the - override layer, and the client serves an overridden flag before it has LaunchDarkly - data. + Returns the override layer when the data system was built with an override source, + and None otherwise. The store served by :attr:`store` overlays this layer. The client + consults the layer directly before it has LaunchDarkly data, so that a flag the layer + holds is served and a flag it does not hold gets the not-ready handling. """ raise NotImplementedError @@ -245,10 +246,10 @@ def store(self) -> AsyncReadOnlyStore: @property @abstractmethod - def override_source_configured(self) -> bool: + def override_layer(self) -> Optional["OverrideLayer"]: """ - Reports whether the data system was built with an override source. The value is - fixed at construction. + Returns the override layer when the data system was built with an override source, + and None otherwise. See :attr:`DataSystem.override_layer`. """ raise NotImplementedError diff --git a/ldclient/impl/datasystem/async_fdv1.py b/ldclient/impl/datasystem/async_fdv1.py index b8b762f1..a7bd1455 100644 --- a/ldclient/impl/datasystem/async_fdv1.py +++ b/ldclient/impl/datasystem/async_fdv1.py @@ -125,9 +125,9 @@ def store(self) -> AsyncReadOnlyStore: return self._store_view @property - def override_source_configured(self) -> bool: + def override_layer(self) -> None: # Overrides are an option of the FDv2 data system only. - return False + return None def set_diagnostic_accumulator(self, diagnostic_accumulator: DiagnosticAccumulator): """ diff --git a/ldclient/impl/datasystem/async_fdv2.py b/ldclient/impl/datasystem/async_fdv2.py index 9f6d93db..22dd735a 100644 --- a/ldclient/impl/datasystem/async_fdv2.py +++ b/ldclient/impl/datasystem/async_fdv2.py @@ -677,8 +677,8 @@ def store(self) -> AsyncReadOnlyStore: return self._store_view @property - def override_source_configured(self) -> bool: - return self._override_source is not None + def override_layer(self) -> Optional[OverrideLayer]: + return self._override_layer async def data_availability(self) -> DataAvailability: # type: ignore[override] """Reports what form of data is currently available, awaiting the store's diff --git a/ldclient/impl/datasystem/fdv1.py b/ldclient/impl/datasystem/fdv1.py index dabaa12d..9d0008a1 100644 --- a/ldclient/impl/datasystem/fdv1.py +++ b/ldclient/impl/datasystem/fdv1.py @@ -234,9 +234,9 @@ def store(self) -> ReadOnlyStore: return self._store_view @property - def override_source_configured(self) -> bool: + def override_layer(self) -> None: # Overrides are an option of the FDv2 data system only. - return False + return None @property def environment_id(self) -> Optional[str]: diff --git a/ldclient/impl/datasystem/fdv2.py b/ldclient/impl/datasystem/fdv2.py index 4d5398fb..b218bc3f 100644 --- a/ldclient/impl/datasystem/fdv2.py +++ b/ldclient/impl/datasystem/fdv2.py @@ -680,8 +680,8 @@ def store(self) -> ReadOnlyStore: return self._store_view @property - def override_source_configured(self) -> bool: - return self._override_source is not None + def override_layer(self) -> Optional[OverrideLayer]: + return self._override_layer __all__ = [ diff --git a/ldclient/testing/mock_components.py b/ldclient/testing/mock_components.py index 59c257e1..f8c2e85e 100644 --- a/ldclient/testing/mock_components.py +++ b/ldclient/testing/mock_components.py @@ -69,15 +69,21 @@ class MockOverrideSource: on every later call to set_overrides, and records its lifecycle calls. """ - def __init__(self, flags: Optional[Dict[str, Any]] = None, segments: Optional[Dict[str, Any]] = None): + def __init__(self, flags: Optional[Dict[str, Any]] = None, segments: Optional[Dict[str, Any]] = None, start_error: Optional[Exception] = None): + """ + :param start_error: when given, start raises it instead of supplying the contents + """ self._flags: Dict[str, Any] = dict(flags or {}) self._segments: Dict[str, Any] = dict(segments or {}) + self._start_error = start_error self._sink: Optional[OverrideSink] = None self.start_count = 0 self.close_count = 0 def start(self, sink: OverrideSink) -> None: self.start_count += 1 + if self._start_error is not None: + raise self._start_error self._sink = sink sink.set_overrides(self._flags, self._segments) diff --git a/ldclient/testing/test_async_client_overrides.py b/ldclient/testing/test_async_client_overrides.py index d8e53bfb..bce576e0 100644 --- a/ldclient/testing/test_async_client_overrides.py +++ b/ldclient/testing/test_async_client_overrides.py @@ -4,15 +4,18 @@ """ import asyncio import threading -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional import pytest from ldclient.async_client import AsyncLDClient from ldclient.async_config import AsyncConfig, AsyncDataSystemConfig +from ldclient.async_feature_store import AsyncInMemoryFeatureStore from ldclient.context import Context +from ldclient.evaluation import EvaluationDetail from ldclient.impl.aio.concurrency import AsyncEvent from ldclient.impl.integrations.files.filedata import make_flag_with_value +from ldclient.interfaces import AsyncFeatureStore, DataStoreMode from ldclient.testing.builders import FlagBuilder from ldclient.testing.mock_async_components import MockAsyncEventProcessor from ldclient.testing.mock_components import ( @@ -21,9 +24,13 @@ MockOverrideSource, StaticInitializer ) +from ldclient.versioned_data_kind import FEATURES user = Context.create('user-key') +# A context whose key is empty is invalid. +invalid_context = Context.create('') + class AsyncHangingSynchronizer: """An async synchronizer that connects but never yields data.""" @@ -49,12 +56,52 @@ async def fetch(self, ss): # type: ignore[override] return super().fetch(ss) +class UnavailableAsyncStore(AsyncInMemoryFeatureStore): + """A persistent store whose every read fails, as during a store outage.""" + + async def is_initialized(self) -> bool: + raise RuntimeError("store unreachable") + + async def get(self, kind, key): + raise RuntimeError("store unreachable") + + async def all(self, kind): + raise RuntimeError("store unreachable") + + +class UninitializedAsyncStore(AsyncInMemoryFeatureStore): + """A persistent store that holds data but was never initialized by an SDK.""" + + @property + def initialized(self) -> bool: + return False + + async def is_initialized(self) -> bool: + return False + + +async def uninitialized_store_holding(key: str) -> UninitializedAsyncStore: + store = UninitializedAsyncStore() + await store.upsert(FEATURES, FlagBuilder(key).version(1).on(False).off_variation(0).variations('ld-value').build().to_json_dict()) + return store + + def single_value_flag(key: str, value: Any) -> dict: return make_flag_with_value(key, value).to_json_dict() -async def make_uninitialized_client(source: MockOverrideSource) -> AsyncLDClient: - datasystem = AsyncDataSystemConfig(synchronizers=[MockDataSourceBuilder(AsyncHangingSynchronizer())], override_source=source.builder) +def recorded_keys(client: AsyncLDClient) -> List[str]: + """The flag keys of the evaluations the client handed to the event processor.""" + processor: Any = client._event_processor + return [event.key for event in processor.events] + + +async def make_uninitialized_client(source: MockOverrideSource, store: Optional[AsyncFeatureStore] = None) -> AsyncLDClient: + """ + A client whose data system can never obtain LaunchDarkly data, with the given override + source and, when given, a read-only persistent store. + """ + datasystem = AsyncDataSystemConfig(synchronizers=[MockDataSourceBuilder(AsyncHangingSynchronizer())], override_source=source.builder, data_store=store, data_store_mode=DataStoreMode.READ_ONLY) config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor()) client = AsyncLDClient(config) await client.start(start_wait=0) @@ -110,6 +157,80 @@ async def test_override_removal_restores_short_circuit(): await client.close() +@pytest.mark.asyncio +@pytest.mark.parametrize('overrides', [{}, {'other-flag': single_value_flag('other-flag', True)}], ids=['empty-layer', 'layer-without-the-key']) +async def test_not_initialized_client_returns_not_ready_for_an_invalid_context_when_the_layer_lacks_the_key(overrides): + client = await make_uninitialized_client(MockOverrideSource(flags=overrides)) + try: + detail = await client.variation_detail('requested-flag', invalid_context, False) + # The not-ready handling runs before the context check, as it does without overrides, + # and records the evaluation of the unknown flag. + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'}) + assert recorded_keys(client) == ['requested-flag'] + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_not_initialized_client_checks_the_context_before_serving_an_override(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source) + try: + detail = await client.variation_detail('overridden-flag', invalid_context, False) + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'USER_NOT_SPECIFIED'}) + assert recorded_keys(client) == [] + finally: + await client.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize('store_outage', [True, False], ids=['store-outage', 'uninitialized-store']) +async def test_not_initialized_client_returns_not_ready_when_the_store_has_no_launchdarkly_data_and_the_layer_lacks_the_key(store_outage): + store: AsyncFeatureStore = UnavailableAsyncStore() if store_outage else await uninitialized_store_holding('requested-flag') + source = MockOverrideSource(flags={'other-flag': single_value_flag('other-flag', True)}) + client = await make_uninitialized_client(source, store=store) + try: + detail = await client.variation_detail('requested-flag', user, False) + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'}) + assert recorded_keys(client) == ['requested-flag'] + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_override_is_served_during_a_persistent_store_outage(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source, store=UnavailableAsyncStore()) + try: + detail = await client.variation_detail('overridden-flag', user, False) + assert detail.value is True + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_all_flags_state_is_invalid_when_the_store_is_uninitialized_and_the_layer_is_empty(): + client = await make_uninitialized_client(MockOverrideSource(), store=await uninitialized_store_holding('ld-flag')) + try: + state = await client.all_flags_state(user) + assert state.valid is False + assert state.to_values_map() == {} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_override_source_start_failure_fails_start_and_closes_the_source(): + source = MockOverrideSource(start_error=RuntimeError("cannot start")) + datasystem = AsyncDataSystemConfig(synchronizers=[MockDataSourceBuilder(AsyncHangingSynchronizer())], override_source=source.builder) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor()) + client = AsyncLDClient(config) + with pytest.raises(RuntimeError): + await client.start(start_wait=0) + assert source.close_count == 1 + + @pytest.mark.asyncio async def test_invalid_override_source_configuration_fails_construction(): datasystem = AsyncDataSystemConfig(synchronizers=[MockDataSourceBuilder(AsyncHangingSynchronizer())], override_source=FailingOverrideSourceBuilder()) diff --git a/ldclient/testing/test_ldclient_overrides.py b/ldclient/testing/test_ldclient_overrides.py index f6b54a44..fea18891 100644 --- a/ldclient/testing/test_ldclient_overrides.py +++ b/ldclient/testing/test_ldclient_overrides.py @@ -4,16 +4,22 @@ """ import logging from queue import Empty, Queue -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional import pytest from ldclient.client import Config, Context, LDClient from ldclient.datasystem import custom from ldclient.evaluation import EvaluationDetail +from ldclient.feature_store import InMemoryFeatureStore from ldclient.impl.datasystem.fdv1 import FDv1 from ldclient.impl.integrations.files.filedata import make_flag_with_value -from ldclient.interfaces import DataSourceState, FlagChange +from ldclient.interfaces import ( + DataSourceState, + DataStoreMode, + FeatureStore, + FlagChange +) from ldclient.testing.builders import ( FlagBuilder, FlagRuleBuilder, @@ -27,18 +33,27 @@ StaticInitializer ) from ldclient.testing.stub_util import MockEventProcessor +from ldclient.versioned_data_kind import FEATURES user = Context.create('user-key') +# A context whose key is empty is invalid. +invalid_context = Context.create('') + def single_value_flag(key: str, value: Any) -> dict: return make_flag_with_value(key, value).to_json_dict() -def make_uninitialized_client(source: MockOverrideSource) -> LDClient: - """A client whose data system can never obtain LaunchDarkly data, with the given override source.""" - datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(source.builder).build() - config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) +def make_uninitialized_client(source: MockOverrideSource, store: Optional[FeatureStore] = None) -> LDClient: + """ + A client whose data system can never obtain LaunchDarkly data, with the given override + source and, when given, a read-only persistent store. + """ + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(source.builder) + if store is not None: + datasystem.data_store(store, DataStoreMode.READ_ONLY) + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem.build(), event_processor_class=MockEventProcessor) return LDClient(config, start_wait=0) @@ -56,6 +71,40 @@ def warnings_containing(caplog, text: str): return [r for r in caplog.records if r.levelno == logging.WARNING and text in r.getMessage()] +def recorded_keys(client: LDClient) -> List[str]: + """The flag keys of the evaluations the client handed to the event processor.""" + processor: Any = client._event_processor + return [event.key for event in processor._events] + + +class UnavailableStore(InMemoryFeatureStore): + """A persistent store whose every read fails, as during a store outage.""" + + @property + def initialized(self) -> bool: + raise RuntimeError("store unreachable") + + def get(self, kind, key, callback=lambda x: x): + raise RuntimeError("store unreachable") + + def all(self, kind, callback=lambda x: x): + raise RuntimeError("store unreachable") + + +class UninitializedStore(InMemoryFeatureStore): + """A persistent store that holds data but was never initialized by an SDK.""" + + @property + def initialized(self) -> bool: + return False + + +def uninitialized_store_holding(key: str) -> UninitializedStore: + store = UninitializedStore() + store.upsert(FEATURES, FlagBuilder(key).version(1).on(False).off_variation(0).variations('ld-value').build().to_json_dict()) + return store + + def test_override_is_served_when_client_is_not_initialized(): source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) with make_uninitialized_client(source) as client: @@ -83,6 +132,48 @@ def test_override_removal_restores_short_circuit(): assert detail.reason == {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'} +@pytest.mark.parametrize('overrides', [{}, {'other-flag': single_value_flag('other-flag', True)}], ids=['empty-layer', 'layer-without-the-key']) +def test_not_initialized_client_returns_not_ready_for_an_invalid_context_when_the_layer_lacks_the_key(overrides): + with make_uninitialized_client(MockOverrideSource(flags=overrides)) as client: + detail = client.variation_detail('requested-flag', invalid_context, False) + # The not-ready handling runs before the context check, as it does without overrides, + # and records the evaluation of the unknown flag. + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'}) + assert recorded_keys(client) == ['requested-flag'] + + +def test_not_initialized_client_checks_the_context_before_serving_an_override(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + detail = client.variation_detail('overridden-flag', invalid_context, False) + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'USER_NOT_SPECIFIED'}) + assert recorded_keys(client) == [] + + +@pytest.mark.parametrize('make_store', [UnavailableStore, lambda: uninitialized_store_holding('requested-flag')], ids=['store-outage', 'uninitialized-store']) +def test_not_initialized_client_returns_not_ready_when_the_store_has_no_launchdarkly_data_and_the_layer_lacks_the_key(make_store): + source = MockOverrideSource(flags={'other-flag': single_value_flag('other-flag', True)}) + with make_uninitialized_client(source, store=make_store()) as client: + detail = client.variation_detail('requested-flag', user, False) + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'}) + assert recorded_keys(client) == ['requested-flag'] + + +def test_override_is_served_during_a_persistent_store_outage(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source, store=UnavailableStore()) as client: + detail = client.variation_detail('overridden-flag', user, False) + assert detail.value is True + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + + +def test_all_flags_state_is_invalid_when_the_store_is_uninitialized_and_the_layer_is_empty(): + with make_uninitialized_client(MockOverrideSource(), store=uninitialized_store_holding('ld-flag')) as client: + state = client.all_flags_state(user) + assert state.valid is False + assert state.to_values_map() == {} + + def test_override_source_is_started_before_the_constructor_returns(): source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) client = make_uninitialized_client(source) @@ -108,6 +199,23 @@ def test_invalid_override_source_configuration_fails_construction(): LDClient(config, start_wait=0) +def test_override_source_start_failure_fails_construction_and_stops_the_started_components(): + stopped = [] + + class RecordingEventProcessor(MockEventProcessor): + def stop(self): + stopped.append(True) + + source = MockOverrideSource(start_error=RuntimeError("cannot start")) + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=RecordingEventProcessor) + with pytest.raises(RuntimeError): + LDClient(config, start_wait=0) + # The source and the event processor that were set up before the failure are closed. + assert source.close_count == 1 + assert stopped == [True] + + def test_override_source_is_not_started_when_offline(): source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) datasystem = custom().overrides(source.builder).build() @@ -117,20 +225,20 @@ def test_override_source_is_not_started_when_offline(): assert client.variation('overridden-flag', user, False) is False -def test_data_system_without_override_source_reports_none_configured(): +def test_data_system_without_override_source_has_no_override_layer(): datasystem = custom().synchronizers(HangingSynchronizer().builder).build() config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) with LDClient(config, start_wait=0) as client: - assert client._data_system.override_source_configured is False + assert client._data_system.override_layer is None detail = client.variation_detail('any-flag', user, False) assert detail.reason == {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'} -def test_legacy_data_system_reports_no_override_source(): +def test_legacy_data_system_has_no_override_layer(): config = Config(sdk_key='SDK_KEY', offline=True) with LDClient(config) as client: assert isinstance(client._data_system, FDv1) - assert client._data_system.override_source_configured is False + assert client._data_system.override_layer is None def test_override_takes_precedence_over_launchdarkly_data(): From cbbe302440ace5f0ba24dd7573340f82e6930085 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:16:43 +0000 Subject: [PATCH 05/26] docs: Describe what the all flags state holds before the client has LaunchDarkly data --- ldclient/async_client.py | 5 +++-- ldclient/client.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ldclient/async_client.py b/ldclient/async_client.py index d5191eca..6f92a086 100644 --- a/ldclient/async_client.py +++ b/ldclient/async_client.py @@ -594,8 +594,9 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState log.warning("all_flags_state() called before client has finished initializing! Using last known values from feature store. This message is logged once.") else: # No data from LaunchDarkly is available. When the override layer holds entries, - # the state is built from them alone. Otherwise the state is unavailable, as it - # is without an override source. + # the state is read through the overlay, which returns the override entries and + # any data the store already holds. Otherwise the state is unavailable, as it is + # without an override source. layer = self._data_system.override_layer if layer is None or layer.is_empty: log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state") diff --git a/ldclient/client.py b/ldclient/client.py index c03de402..8d0996d0 100644 --- a/ldclient/client.py +++ b/ldclient/client.py @@ -584,8 +584,9 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: log.warning("all_flags_state() called before client has finished initializing! Using last known values from feature store. This message is logged once.") else: # No data from LaunchDarkly is available. When the override layer holds entries, - # the state is built from them alone. Otherwise the state is unavailable, as it - # is without an override source. + # the state is read through the overlay, which returns the override entries and + # any data the store already holds. Otherwise the state is unavailable, as it is + # without an override source. layer = self._data_system.override_layer if layer is None or layer.is_empty: log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state") From 7cc127e208d1dc61606935eb29637165a8391f6b Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Fri, 16 Oct 2026 21:31:29 +0000 Subject: [PATCH 06/26] feat: Notify override flags when the LaunchDarkly data they depend on changes The override layer keeps a dependency graph of its own entries, rebuilt on each replacement. The store expands every changed LaunchDarkly item through both its own graph and the layer's, to a fixpoint, so a change to a prerequisite flag or a segment that an override entry references notifies that entry's flag as well. This matches how dependent flags behave for LaunchDarkly data. --- ldclient/impl/datasystem/async_fdv2.py | 1 + ldclient/impl/datasystem/fdv2.py | 4 + ldclient/impl/datasystem/store.py | 50 ++++++- ldclient/impl/dependency_tracker.py | 21 ++- ldclient/impl/overrides/layer.py | 27 +++- .../test_store_external_dependencies.py | 137 ++++++++++++++++++ ldclient/testing/impl/test_overrides.py | 33 +++++ ldclient/testing/test_ldclient_overrides.py | 20 +++ 8 files changed, 285 insertions(+), 8 deletions(-) create mode 100644 ldclient/testing/impl/datasystem/test_store_external_dependencies.py diff --git a/ldclient/impl/datasystem/async_fdv2.py b/ldclient/impl/datasystem/async_fdv2.py index 22dd735a..36fcf999 100644 --- a/ldclient/impl/datasystem/async_fdv2.py +++ b/ldclient/impl/datasystem/async_fdv2.py @@ -287,6 +287,7 @@ def __init__( self._override_source = data_system_config.override_source.build(config) self._override_layer = OverrideLayer() self._overlay = AsyncOverrideStoreView(self._store_view, self._override_layer) + self._store.set_external_dependencies(self._override_layer) # Concurrency self._stop_event = AsyncEvent() diff --git a/ldclient/impl/datasystem/fdv2.py b/ldclient/impl/datasystem/fdv2.py index b218bc3f..f51ccd01 100644 --- a/ldclient/impl/datasystem/fdv2.py +++ b/ldclient/impl/datasystem/fdv2.py @@ -277,6 +277,10 @@ def __init__( self._override_source = data_system_config.override_source.build(config) self._override_layer = OverrideLayer() self._overlay = OverrideStoreView(self._store_view, self._override_layer) + # A LaunchDarkly change to a flag or segment that an override entry depends on + # must notify that entry's flag as well. The store reaches those entries through + # the layer's dependency graph. + self._store.set_external_dependencies(self._override_layer) def _create_store(self, flag_change_listeners: Listeners, change_set_listeners: Listeners) -> Store: return Store(flag_change_listeners, change_set_listeners) diff --git a/ldclient/impl/datasystem/store.py b/ldclient/impl/datasystem/store.py index b6532605..3904312b 100644 --- a/ldclient/impl/datasystem/store.py +++ b/ldclient/impl/datasystem/store.py @@ -10,7 +10,11 @@ from collections import defaultdict from typing import Any, Callable, Dict, List, Optional, Set -from ldclient.impl.dependency_tracker import DependencyTracker, KindAndKey +from ldclient.impl.dependency_tracker import ( + DependencyTracker, + ExternalDependencies, + KindAndKey +) from ldclient.impl.listeners import Listeners from ldclient.impl.model.entity import ModelEntity from ldclient.impl.rwlock import ReadWriteLock @@ -182,6 +186,10 @@ def __init__( # Used to track dependencies between items in the store self._dependency_tracker = DependencyTracker() + # A second dependency graph, such as the override layer's, whose items may depend on + # items in this store. Set once at construction of the data system, before any data. + self._external_dependencies: Optional[ExternalDependencies] = None + # Listeners for events self._flag_change_listeners = flag_change_listeners self._change_set_listeners = change_set_listeners @@ -195,6 +203,40 @@ def __init__( # Thread synchronization self._lock = threading.RLock() + def set_external_dependencies(self, dependencies: Optional[ExternalDependencies]) -> None: + """ + Registers a second dependency graph whose items may depend on items in this store. A + change to an item in this store then also notifies the external items that depend on + it, directly or through any chain of dependencies in either graph. + """ + with self._lock: + self._external_dependencies = dependencies + + def _add_affected_items(self, items_out: Set[KindAndKey], initial_modified_item: KindAndKey) -> None: + """ + Adds the item and everything that depends on it to the set, following this store's + dependency graph and the external one. Every item reached through either graph is + expanded through both, so a dependency chain may cross between them any number of times. + """ + external = self._external_dependencies + if external is None: + self._dependency_tracker.add_affected_items(items_out, initial_modified_item) + return + + pending: List[KindAndKey] = [initial_modified_item] + while pending: + current = pending.pop() + # Each graph gets its own working set: a tracker does not expand an item that is + # already present, so sharing one set would stop the second graph at the start. + via_store: Set[KindAndKey] = set() + self._dependency_tracker.add_affected_items(via_store, current) + via_external: Set[KindAndKey] = set() + external.add_affected_items(via_external, current) + for reached in via_store | via_external: + if reached not in items_out: + items_out.add(reached) + pending.append(reached) + def selector(self) -> Selector: """Returns the current selector.""" with self._lock: @@ -282,7 +324,7 @@ def _apply_delta( kind, key, item ) if has_listeners: - self._dependency_tracker.add_affected_items( + self._add_affected_items( affected_items, KindAndKey(kind=kind, key=key) ) @@ -353,11 +395,11 @@ def _compute_changed_items_for_full_data_set( # If either is missing or versions differ, it's a change if old_item is None or new_item is None: - self._dependency_tracker.add_affected_items( + self._add_affected_items( affected_items, KindAndKey(kind=kind, key=key) ) elif old_item.get("version", 0) != new_item.get("version", 0): - self._dependency_tracker.add_affected_items( + self._add_affected_items( affected_items, KindAndKey(kind=kind, key=key) ) diff --git a/ldclient/impl/dependency_tracker.py b/ldclient/impl/dependency_tracker.py index 23d6b0d5..10601862 100644 --- a/ldclient/impl/dependency_tracker.py +++ b/ldclient/impl/dependency_tracker.py @@ -1,6 +1,7 @@ -from typing import Dict, List, NamedTuple, Optional, Set, Union +from typing import Dict, List, NamedTuple, Optional, Protocol, Set, Union from ldclient.impl.model.clause import Clause +from ldclient.impl.model.entity import ModelEntity from ldclient.impl.model.feature_flag import FeatureFlag from ldclient.impl.model.segment import Segment from ldclient.versioned_data_kind import FEATURES, SEGMENTS, VersionedDataKind @@ -11,6 +12,20 @@ class KindAndKey(NamedTuple): key: str +class ExternalDependencies(Protocol): + """ + A second dependency graph whose items may depend on items held elsewhere, such as the + override layer, whose entries may depend on LaunchDarkly data. A store that is handed one + extends its change notifications to the external items that depend on a changed item. + """ + + def add_affected_items(self, items_out: Set[KindAndKey], initial_modified_item: KindAndKey) -> None: + """ + Adds the given item and every item in this graph that directly or indirectly depends on + it to the set. Items already in the set are not expanded again. + """ + + class DependencyTracker: """ The DependencyTracker is responsible for tracking both up and downstream @@ -23,7 +38,7 @@ def __init__(self): self.__children: Dict[KindAndKey, Set[KindAndKey]] = {} self.__parents: Dict[KindAndKey, Set[KindAndKey]] = {} - def update_dependencies_from(self, from_kind: VersionedDataKind, from_key: str, from_item: Optional[Union[dict, FeatureFlag, Segment]]): + def update_dependencies_from(self, from_kind: VersionedDataKind, from_key: str, from_item: Optional[Union[dict, ModelEntity]]): """ Updates the dependency graph when an item has changed. @@ -83,7 +98,7 @@ def reset(self): self.__parents.clear() @staticmethod - def compute_dependencies_from(from_kind: VersionedDataKind, from_item: Optional[Union[dict, FeatureFlag, Segment]]) -> Set[KindAndKey]: + def compute_dependencies_from(from_kind: VersionedDataKind, from_item: Optional[Union[dict, ModelEntity]]) -> Set[KindAndKey]: """ @param from_kind [String] @param from_item [LaunchDarkly::Impl::Model::FeatureFlag, LaunchDarkly::Impl::Model::Segment] diff --git a/ldclient/impl/overrides/layer.py b/ldclient/impl/overrides/layer.py index d03dc7af..1ba5ecd3 100644 --- a/ldclient/impl/overrides/layer.py +++ b/ldclient/impl/overrides/layer.py @@ -3,8 +3,9 @@ segment key, and is replaced wholesale on each update. """ -from typing import Any, Dict, Mapping, Optional, Tuple +from typing import Any, Dict, Mapping, Optional, Set, Tuple +from ldclient.impl.dependency_tracker import DependencyTracker, KindAndKey from ldclient.impl.model import ModelEntity from ldclient.impl.rwlock import ReadWriteLock from ldclient.versioned_data_kind import FEATURES, SEGMENTS, VersionedDataKind @@ -26,6 +27,10 @@ class OverrideLayer: def __init__(self): self._lock = ReadWriteLock() self._contents: LayerContents = _empty_contents() + # The dependency graph of the override entries, rebuilt with each replacement. The store + # consults it so that a LaunchDarkly change to a prerequisite or a segment also notifies + # the override entries that depend on it. + self._dependencies = DependencyTracker() # Read without the lock on every evaluation. A single attribute read is atomic, so the # per-evaluation cost of a configured but unpopulated override layer is negligible. self._non_empty = False @@ -45,12 +50,32 @@ def set_all(self, flags: Mapping[str, Any], segments: Mapping[str, Any]) -> Tupl SEGMENTS: {key: SEGMENTS.decode(item).with_override_marker() for key, item in segments.items()}, } count = len(replacement[FEATURES]) + len(replacement[SEGMENTS]) + dependencies = DependencyTracker() + for kind, items in replacement.items(): + for key, item in items.items(): + try: + dependencies.update_dependencies_from(kind, key, item) + except Exception: + # An entry whose dependency fields cannot be read contributes no edges. It is + # still served, and the remaining entries are still tracked. + pass with self._lock.write(): previous = self._contents self._contents = replacement + self._dependencies = dependencies self._non_empty = count > 0 return previous, replacement + def add_affected_items(self, items_out: Set[KindAndKey], initial_modified_item: KindAndKey) -> None: + """ + Adds the given item and every override entry that directly or indirectly depends on it + to the set. The item itself need not be an override entry: a LaunchDarkly flag or + segment that an override entry references is a valid starting point. + """ + with self._lock.read(): + dependencies = self._dependencies + dependencies.add_affected_items(items_out, initial_modified_item) + def get(self, kind: VersionedDataKind, key: str) -> Optional[ModelEntity]: """Returns the override entry for a key, or None when the layer has none.""" if not self._non_empty: diff --git a/ldclient/testing/impl/datasystem/test_store_external_dependencies.py b/ldclient/testing/impl/datasystem/test_store_external_dependencies.py new file mode 100644 index 00000000..87422a6a --- /dev/null +++ b/ldclient/testing/impl/datasystem/test_store_external_dependencies.py @@ -0,0 +1,137 @@ +""" +Tests for the store's use of a second dependency graph: a LaunchDarkly change to a flag or +segment that an override entry depends on notifies the override entry's flag as well. +""" +from typing import List, Optional + +from ldclient.impl.datasystem.store import Store +from ldclient.impl.dependency_tracker import KindAndKey +from ldclient.impl.listeners import Listeners +from ldclient.impl.overrides import OverrideLayer +from ldclient.interfaces import ( + Change, + ChangeSet, + ChangeType, + FlagChange, + IntentCode, + ObjectKind, + Selector +) +from ldclient.testing.builders import ( + FlagBuilder, + FlagRuleBuilder, + SegmentBuilder, + make_clause_matching_segment_key +) +from ldclient.versioned_data_kind import FEATURES, SEGMENTS + + +def _flag(key: str, version: int, prerequisite: Optional[str] = None, segment: Optional[str] = None) -> dict: + builder = FlagBuilder(key).version(version).on(True).variations(True, False).fallthrough_variation(1) + if prerequisite is not None: + builder.prerequisite(prerequisite, 0) + if segment is not None: + builder.rules(FlagRuleBuilder().id('r').variation(0).clauses(make_clause_matching_segment_key(segment)).build()) + return builder.build().to_json_dict() + + +def _segment(key: str, version: int) -> dict: + return SegmentBuilder(key).version(version).build().to_json_dict() + + +def _put(kind: ObjectKind, item: dict) -> Change: + return Change(action=ChangeType.PUT, kind=kind, key=item['key'], version=item['version'], object=item) + + +def _full(*changes: Change) -> ChangeSet: + return ChangeSet(intent_code=IntentCode.TRANSFER_FULL, changes=list(changes), selector=Selector.no_selector()) + + +def _delta(*changes: Change) -> ChangeSet: + return ChangeSet(intent_code=IntentCode.TRANSFER_CHANGES, changes=list(changes), selector=Selector.no_selector()) + + +class Fixture: + def __init__(self, layer: Optional[OverrideLayer] = None): + self.notified: List[str] = [] + listeners = Listeners() + listeners.add(lambda change: self.notified.append(change.key)) + self.store = Store(listeners, Listeners()) + if layer is not None: + self.store.set_external_dependencies(layer) + + def take_notified(self) -> List[str]: + result = sorted(self.notified) + del self.notified[:] + return result + + +def test_delta_to_a_prerequisite_notifies_the_override_flag_that_depends_on_it(): + layer = OverrideLayer() + layer.set_all({'override-flag': _flag('override-flag', 1, prerequisite='ld-flag')}, {}) + f = Fixture(layer) + f.store.apply(_full(_put(ObjectKind.FLAG, _flag('ld-flag', 1))), False) + f.take_notified() + + f.store.apply(_delta(_put(ObjectKind.FLAG, _flag('ld-flag', 2))), False) + assert f.take_notified() == ['ld-flag', 'override-flag'] + + +def test_full_data_set_replacement_notifies_the_override_flags_that_depend_on_changed_items(): + layer = OverrideLayer() + layer.set_all({'override-flag': _flag('override-flag', 1, segment='ld-segment')}, {}) + f = Fixture(layer) + f.store.apply(_full(_put(ObjectKind.SEGMENT, _segment('ld-segment', 1)), _put(ObjectKind.FLAG, _flag('unrelated', 1))), False) + f.take_notified() + + f.store.apply(_full(_put(ObjectKind.SEGMENT, _segment('ld-segment', 2)), _put(ObjectKind.FLAG, _flag('unrelated', 1))), False) + # The segment is not a flag, so only the dependent override flag is notified. + assert f.take_notified() == ['override-flag'] + + +def test_dependency_chain_crosses_between_the_store_and_the_layer(): + # LaunchDarkly flag -> override flag -> LaunchDarkly segment: the chain enters the layer + # and comes back out, and a segment change must reach the LaunchDarkly flag at the top. + layer = OverrideLayer() + layer.set_all({'override-flag': _flag('override-flag', 1, segment='ld-segment')}, {}) + f = Fixture(layer) + f.store.apply(_full( + _put(ObjectKind.SEGMENT, _segment('ld-segment', 1)), + _put(ObjectKind.FLAG, _flag('ld-flag', 1, prerequisite='override-flag')), + ), False) + f.take_notified() + + f.store.apply(_delta(_put(ObjectKind.SEGMENT, _segment('ld-segment', 2))), False) + assert f.take_notified() == ['ld-flag', 'override-flag'] + + +def test_layer_replacement_changes_which_flags_a_store_change_notifies(): + layer = OverrideLayer() + layer.set_all({'override-flag': _flag('override-flag', 1, prerequisite='ld-flag')}, {}) + f = Fixture(layer) + f.store.apply(_full(_put(ObjectKind.FLAG, _flag('ld-flag', 1))), False) + f.take_notified() + + layer.set_all({}, {}) + f.store.apply(_delta(_put(ObjectKind.FLAG, _flag('ld-flag', 2))), False) + assert f.take_notified() == ['ld-flag'] + + +def test_store_without_external_dependencies_notifies_only_its_own_dependents(): + f = Fixture() + f.store.apply(_full(_put(ObjectKind.FLAG, _flag('ld-flag', 1)), _put(ObjectKind.FLAG, _flag('top', 1, prerequisite='ld-flag'))), False) + f.take_notified() + + f.store.apply(_delta(_put(ObjectKind.FLAG, _flag('ld-flag', 2))), False) + assert f.take_notified() == ['ld-flag', 'top'] + + +def test_external_dependencies_do_not_hide_the_store_s_own_dependents(): + layer = OverrideLayer() + layer.set_all({'override-flag': _flag('override-flag', 1, prerequisite='ld-flag')}, {}) + f = Fixture(layer) + f.store.apply(_full(_put(ObjectKind.FLAG, _flag('ld-flag', 1)), _put(ObjectKind.FLAG, _flag('top', 1, prerequisite='ld-flag'))), False) + f.take_notified() + + f.store.apply(_delta(_put(ObjectKind.FLAG, _flag('ld-flag', 2))), False) + assert f.take_notified() == ['ld-flag', 'override-flag', 'top'] diff --git a/ldclient/testing/impl/test_overrides.py b/ldclient/testing/impl/test_overrides.py index 6da55c2d..58502102 100644 --- a/ldclient/testing/impl/test_overrides.py +++ b/ldclient/testing/impl/test_overrides.py @@ -7,6 +7,7 @@ import pytest from ldclient.impl.datasystem.store import InMemoryFeatureStore +from ldclient.impl.dependency_tracker import KindAndKey from ldclient.impl.overrides import ( AsyncOverrideStoreView, OverrideLayer, @@ -153,6 +154,38 @@ def test_layer_all_returns_entries_of_a_kind(): # Overlay # --------------------------------------------------------------------------- +def test_layer_add_affected_items_reaches_the_entries_that_depend_on_an_item(): + top = FlagBuilder('top').version(1).on(True).variations(True, False).fallthrough_variation(0).prerequisite('mid', 0).build() + mid = FlagBuilder('mid').version(1).on(True).variations(True, False).fallthrough_variation(1).rules( + FlagRuleBuilder().id('r').variation(0).clauses(make_clause_matching_segment_key('seg')).build() + ).build() + layer = OverrideLayer() + layer.set_all({'top': top, 'mid': mid, 'other': flag('other')}, {}) + + # The segment is not an override entry; the chain still starts from it. + affected = set() + layer.add_affected_items(affected, KindAndKey(kind=SEGMENTS, key='seg')) + assert affected == {KindAndKey(SEGMENTS, 'seg'), KindAndKey(FEATURES, 'mid'), KindAndKey(FEATURES, 'top')} + + affected = set() + layer.add_affected_items(affected, KindAndKey(kind=FEATURES, key='other')) + assert affected == {KindAndKey(FEATURES, 'other')} + + +def test_layer_dependency_graph_follows_each_replacement(): + top = FlagBuilder('top').version(1).on(True).variations(True, False).fallthrough_variation(0).prerequisite('leaf', 0).build() + layer = OverrideLayer() + layer.set_all({'top': top}, {}) + affected = set() + layer.add_affected_items(affected, KindAndKey(kind=FEATURES, key='leaf')) + assert affected == {KindAndKey(FEATURES, 'leaf'), KindAndKey(FEATURES, 'top')} + + layer.set_all({}, {}) + affected = set() + layer.add_affected_items(affected, KindAndKey(kind=FEATURES, key='leaf')) + assert affected == {KindAndKey(FEATURES, 'leaf')} + + def test_overlay_get_precedence(): base = FakeBaseStore(flags={'both': flag('both', 1), 'base-only': flag('base-only', 1)}) layer = OverrideLayer() diff --git a/ldclient/testing/test_ldclient_overrides.py b/ldclient/testing/test_ldclient_overrides.py index fea18891..95017a21 100644 --- a/ldclient/testing/test_ldclient_overrides.py +++ b/ldclient/testing/test_ldclient_overrides.py @@ -14,6 +14,7 @@ from ldclient.feature_store import InMemoryFeatureStore from ldclient.impl.datasystem.fdv1 import FDv1 from ldclient.impl.integrations.files.filedata import make_flag_with_value +from ldclient.integrations.test_datav2 import TestDataV2 from ldclient.interfaces import ( DataSourceState, DataStoreMode, @@ -338,6 +339,25 @@ def test_flag_tracker_is_notified_of_override_changes(): assert changes.get(timeout=5).key == 'overridden-flag' +def test_launchdarkly_change_to_a_prerequisite_notifies_the_override_flag_that_depends_on_it(): + td = TestDataV2.data_source() + td.update(td.flag('ld-flag').on(True)) + override_flag = FlagBuilder('override-flag').version(1).on(True).variations(True, False).fallthrough_variation(0).prerequisite('ld-flag', 0).build().to_json_dict() + source = MockOverrideSource({'override-flag': override_flag}, {}) + datasystem = custom().synchronizers(td.builder).overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + with LDClient(config, start_wait=5) as client: + assert client.is_initialized() is True + changes: Queue = Queue() + client.flag_tracker.add_listener(lambda change: changes.put(change)) + + td.update(td.flag('ld-flag').on(False)) + keys = {changes.get(timeout=5).key, changes.get(timeout=5).key} + assert keys == {'ld-flag', 'override-flag'} + with pytest.raises(Empty): + changes.get(timeout=0.3) + + def test_flag_value_change_listener_sees_override_value_changes(): ld_flag = FlagBuilder('flag').version(100).on(False).off_variation(0).variations('ld-value').build().to_json_dict() source = MockOverrideSource() From c1d4f8dcc5d195a35b6f2f3f7a642e3a5f693d15 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:08:57 +0000 Subject: [PATCH 07/26] fix: Build the overrides-only flag state from the layer and keep a failed override source from stopping the client all_flags_state returns a state of overrides alone only when no LaunchDarkly data is available and the override layer holds flags, and it reads those flags from the layer itself. Before, any layer entry was enough: a layer of segments alone over a store that held LaunchDarkly data without an init marker produced a valid state of that data, and with override flags present the same data entered the state. The overlay no longer serves the layer's entries alone when the base store fails to list its items, so a client whose store fails reports an invalid state and logs the failure, as it does without a source. An exception from the override source's start no longer fails client construction or start. The data system logs it, closes the source, clears what the source supplied, and the client runs without overrides. The SDK's own file-based source does not raise from start; a custom source could. Stopping the data system closes the override sink before it closes the source, so a snapshot supplied during shutdown is ignored. When the client is offline and an override source is configured, the SDK logs a warning that the source is not started. --- ldclient/async_client.py | 39 ++++---- ldclient/async_config.py | 3 + ldclient/client.py | 37 ++++--- ldclient/config.py | 3 + ldclient/datasystem.py | 3 + ldclient/impl/datasystem/async_fdv2.py | 39 ++++++-- ldclient/impl/datasystem/fdv2.py | 39 ++++++-- ldclient/impl/overrides/overlay.py | 32 ++---- ldclient/impl/overrides/sink.py | 46 +++++++-- ldclient/interfaces.py | 9 +- ldclient/testing/impl/test_overrides.py | 63 ++++++++++-- ldclient/testing/mock_components.py | 18 +++- .../testing/test_async_client_overrides.py | 95 +++++++++++++++++- ldclient/testing/test_ldclient_overrides.py | 98 ++++++++++++++++--- 14 files changed, 412 insertions(+), 112 deletions(-) diff --git a/ldclient/async_client.py b/ldclient/async_client.py index 6f92a086..e4759f90 100644 --- a/ldclient/async_client.py +++ b/ldclient/async_client.py @@ -4,7 +4,7 @@ import asyncio import traceback -from typing import Any, Callable, List, Optional, Tuple +from typing import Any, Callable, Dict, List, Optional, Tuple from uuid import uuid4 from ldclient.async_config import AsyncConfig @@ -585,7 +585,7 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState log.warning("all_flags_state() called, but client is in offline mode. Returning empty state") return FeatureFlagsState(False) - overrides_only = False + override_flags: Optional[Dict[str, Any]] = None availability = await self._data_system.data_availability() if availability != DataAvailability.REFRESHED: if availability == DataAvailability.CACHED: @@ -593,15 +593,16 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState self._all_flags_cached_data_warned = True log.warning("all_flags_state() called before client has finished initializing! Using last known values from feature store. This message is logged once.") else: - # No data from LaunchDarkly is available. When the override layer holds entries, - # the state is read through the overlay, which returns the override entries and - # any data the store already holds. Otherwise the state is unavailable, as it is - # without an override source. + # No data from LaunchDarkly is available. When the override layer holds flags, + # the state contains only those, read from the layer itself: a store that is not + # initialized can still hold LaunchDarkly data, which the state must not include. + # Without override flags the state is unavailable, as it is without an override + # source. layer = self._data_system.override_layer - if layer is None or layer.is_empty: + override_flags = layer.all(FEATURES) if layer is not None else {} + if len(override_flags) == 0: log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state") return FeatureFlagsState(False) - overrides_only = True if not context.valid: log.warning("Context was invalid for all_flags_state (%s); returning default value" % context.error) @@ -611,21 +612,19 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState client_only = kwargs.get('client_side_only', False) with_reasons = kwargs.get('with_reasons', False) details_only_if_tracked = kwargs.get('details_only_for_tracked_flags', False) - try: - flags_map = await self._data_system.store.all(FEATURES) - if flags_map is None: - raise ValueError("feature store error") - except Exception as e: - log.error("Unable to read flags for all_flag_state: %s" % repr(e)) - return FeatureFlagsState(False) - - if overrides_only: - if len(flags_map) == 0: - log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state") - return FeatureFlagsState(False) + if override_flags is not None: if not self._all_flags_overrides_only_warned: self._all_flags_overrides_only_warned = True log.warning("all_flags_state() called before client has finished initializing! Returning only flags from the override layer. This message is logged once.") + flags_map = override_flags + else: + try: + flags_map = await self._data_system.store.all(FEATURES) + if flags_map is None: + raise ValueError("feature store error") + except Exception as e: + log.error("Unable to read flags for all_flag_state: %s" % repr(e)) + return FeatureFlagsState(False) for key, flag in flags_map.items(): if client_only and not flag.get('clientSide', False): diff --git a/ldclient/async_config.py b/ldclient/async_config.py index 8537d02f..c64d0689 100644 --- a/ldclient/async_config.py +++ b/ldclient/async_config.py @@ -133,6 +133,9 @@ class AsyncDataSystemConfig: An optional override source. Its flag and segment definitions take precedence over LaunchDarkly data at evaluation time. Flag overrides are currently experimental and subject to change. + + When the client is offline, the source is not built or started: no overrides are served, and + a configuration error in the source is not reported. The SDK logs a warning instead. """ diff --git a/ldclient/client.py b/ldclient/client.py index 8d0996d0..d7796369 100644 --- a/ldclient/client.py +++ b/ldclient/client.py @@ -573,7 +573,7 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: self._check_forked() - overrides_only = False + override_flags: Optional[Dict[str, Any]] = None availability = self._data_system.data_availability if availability != DataAvailability.REFRESHED: if availability == DataAvailability.CACHED: @@ -583,15 +583,16 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: self._all_flags_cached_data_warned = True log.warning("all_flags_state() called before client has finished initializing! Using last known values from feature store. This message is logged once.") else: - # No data from LaunchDarkly is available. When the override layer holds entries, - # the state is read through the overlay, which returns the override entries and - # any data the store already holds. Otherwise the state is unavailable, as it is - # without an override source. + # No data from LaunchDarkly is available. When the override layer holds flags, + # the state contains only those, read from the layer itself: a store that is not + # initialized can still hold LaunchDarkly data, which the state must not include. + # Without override flags the state is unavailable, as it is without an override + # source. layer = self._data_system.override_layer - if layer is None or layer.is_empty: + override_flags = layer.all(FEATURES) if layer is not None else {} + if len(override_flags) == 0: log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state") return FeatureFlagsState(False) - overrides_only = True if not context.valid: log.warning("Context was invalid for all_flags_state (%s); returning default value" % context.error) @@ -601,23 +602,21 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: client_only = kwargs.get('client_side_only', False) with_reasons = kwargs.get('with_reasons', False) details_only_if_tracked = kwargs.get('details_only_for_tracked_flags', False) - try: - flags_map = self._data_system.store.all(FEATURES, lambda x: x) - if flags_map is None: - raise ValueError("feature store error") - except Exception as e: - log.error("Unable to read flags for all_flag_state: %s" % repr(e)) - return FeatureFlagsState(False) - - if overrides_only: - if len(flags_map) == 0: - log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state") - return FeatureFlagsState(False) + if override_flags is not None: if not self._all_flags_overrides_only_warned: with self._cached_data_warning_lock: if not self._all_flags_overrides_only_warned: self._all_flags_overrides_only_warned = True log.warning("all_flags_state() called before client has finished initializing! Returning only flags from the override layer. This message is logged once.") + flags_map = override_flags + else: + try: + flags_map = self._data_system.store.all(FEATURES, lambda x: x) + if flags_map is None: + raise ValueError("feature store error") + except Exception as e: + log.error("Unable to read flags for all_flag_state: %s" % repr(e)) + return FeatureFlagsState(False) for key, flag in flags_map.items(): if client_only and not flag.get('clientSide', False): diff --git a/ldclient/config.py b/ldclient/config.py index b6bb7ff5..4fb2b572 100644 --- a/ldclient/config.py +++ b/ldclient/config.py @@ -301,6 +301,9 @@ class DataSystemConfig: An optional override source. Its flag and segment definitions take precedence over LaunchDarkly data at evaluation time. Flag overrides are currently experimental and subject to change. + + When the client is offline, the source is not built or started: no overrides are served, and + a configuration error in the source is not reported. The SDK logs a warning instead. """ diff --git a/ldclient/datasystem.py b/ldclient/datasystem.py index 14b95a7e..0e36ec34 100644 --- a/ldclient/datasystem.py +++ b/ldclient/datasystem.py @@ -99,6 +99,9 @@ def overrides(self, source: OverrideSourceBuilder) -> "ConfigBuilder": initialization status or data source status. Configuring it changes nothing until the source actually supplies an override. At most one override source is accepted. + When the client is offline, the source is not built or started: no overrides are served, + and a configuration error in the source is not reported. The SDK logs a warning instead. + Example, with the file-based source and the SDK's default data sources: :: diff --git a/ldclient/impl/datasystem/async_fdv2.py b/ldclient/impl/datasystem/async_fdv2.py index 36fcf999..7592d02c 100644 --- a/ldclient/impl/datasystem/async_fdv2.py +++ b/ldclient/impl/datasystem/async_fdv2.py @@ -282,12 +282,15 @@ def __init__( # source configuration a construction error. self._override_source: Optional[OverrideSource] = None self._override_layer: Optional[OverrideLayer] = None + self._override_sink: Optional[OverrideSinkImpl] = None self._overlay: Optional[AsyncOverrideStoreView] = None if data_system_config.override_source is not None and not self._disabled: self._override_source = data_system_config.override_source.build(config) self._override_layer = OverrideLayer() self._overlay = AsyncOverrideStoreView(self._store_view, self._override_layer) self._store.set_external_dependencies(self._override_layer) + elif data_system_config.override_source is not None: + log.warning("Offline mode is enabled; the override source is not started and no overrides are served") # Concurrency self._stop_event = AsyncEvent() @@ -346,8 +349,19 @@ def start(self, set_on_ready: AsyncEvent): def notify(key: str) -> None: loop.call_soon_threadsafe(listeners.notify, FlagChange(key)) - sink = OverrideSinkImpl(self._override_layer, self._store._memory_store, notify, listeners.has_listeners) - self._override_source.start(sink) + self._override_sink = OverrideSinkImpl(self._override_layer, self._store._memory_store, notify, listeners.has_listeners) + try: + self._override_source.start(self._override_sink) + except Exception as e: + # The SDK's own file-based source does not raise from start; it logs a failed + # load and retries it. A custom source could, and a client in an incident is + # better served by running without its overrides than by not starting. + log.error("Unable to start the override source; the client runs without overrides: %s", e) + self._close_override_source("Unable to close the override source after it failed to start") + # The source may have supplied a snapshot before it failed. Clearing the layer + # removes it, so the client runs without overrides as logged, and a late + # snapshot from the closed source is ignored. + self._override_sink.close_and_clear() # Start the main coordination loop self._runner.spawn("AsyncFDv2-main", lambda: self._run_main_loop(set_on_ready)) @@ -355,10 +369,11 @@ def notify(key: str) -> None: async def stop(self): """Stop the AsyncFDv2 data system and all the work it is coordinating.""" if self._override_source is not None: - try: - self._override_source.close() - except Exception as e: - log.error("Error closing the override source: %s", e) + # The sink stops accepting snapshots first, so a snapshot that the source supplies + # while it shuts down cannot update the layer or notify listeners. + if self._override_sink is not None: + self._override_sink.close() + self._close_override_source("Unable to close the override source") self._stop_event.set() @@ -375,6 +390,18 @@ async def stop(self): # Close the store await self._store.close() + def _close_override_source(self, failure_message: str) -> None: + # A custom source can raise from close. That is logged and not raised, so the rest of + # the client still stops, or still starts after a failed start. + source = self._override_source + self._override_source = None + if source is None: + return + try: + source.close() + except Exception as e: + log.error("%s: %s", failure_message, e) + async def _run_main_loop(self, set_on_ready: AsyncEvent): """Main coordination loop that manages initializers and synchronizers.""" try: diff --git a/ldclient/impl/datasystem/fdv2.py b/ldclient/impl/datasystem/fdv2.py index f51ccd01..2e034001 100644 --- a/ldclient/impl/datasystem/fdv2.py +++ b/ldclient/impl/datasystem/fdv2.py @@ -272,6 +272,7 @@ def __init__( # invalid component configuration. self._override_source: Optional[OverrideSource] = None self._override_layer: Optional[OverrideLayer] = None + self._override_sink: Optional[OverrideSinkImpl] = None self._overlay: Optional[OverrideStoreView] = None if data_system_config.override_source is not None and not self._disabled: self._override_source = data_system_config.override_source.build(config) @@ -281,6 +282,8 @@ def __init__( # must notify that entry's flag as well. The store reaches those entries through # the layer's dependency graph. self._store.set_external_dependencies(self._override_layer) + elif data_system_config.override_source is not None: + log.warning("Offline mode is enabled; the override source is not started and no overrides are served") def _create_store(self, flag_change_listeners: Listeners, change_set_listeners: Listeners) -> Store: return Store(flag_change_listeners, change_set_listeners) @@ -316,13 +319,24 @@ def start(self, set_on_ready: Event): if self._override_source is not None and self._override_layer is not None: # The source starts before the main thread, so a source that loads synchronously # has its overrides in place before the client begins evaluating. - sink = OverrideSinkImpl( + self._override_sink = OverrideSinkImpl( self._override_layer, self._store_view, lambda key: self._flag_change_listeners.notify(FlagChange(key)), self._flag_change_listeners.has_listeners, ) - self._override_source.start(sink) + try: + self._override_source.start(self._override_sink) + except Exception as e: + # The SDK's own file-based source does not raise from start; it logs a failed + # load and retries it. A custom source could, and a client in an incident is + # better served by running without its overrides than by not starting. + log.error("Unable to start the override source; the client runs without overrides: %s", e) + self._close_override_source("Unable to close the override source after it failed to start") + # The source may have supplied a snapshot before it failed. Clearing the layer + # removes it, so the client runs without overrides as logged, and a late + # snapshot from the closed source is ignored. + self._override_sink.close_and_clear() # Start the main coordination thread main_thread = Thread( @@ -337,10 +351,11 @@ def start(self, set_on_ready: Event): def stop(self): """Stop the FDv2 data system and all associated threads.""" if self._override_source is not None: - try: - self._override_source.close() - except Exception as e: - log.error("Error closing the override source: %s", e) + # The sink stops accepting snapshots first, so a snapshot that the source supplies + # while it shuts down cannot update the layer or notify listeners. + if self._override_sink is not None: + self._override_sink.close() + self._close_override_source("Unable to close the override source") self._stop_event.set() @@ -667,6 +682,18 @@ def _record_environment_id(self, environment_id: Optional[str]): with self._lock.write(): self._environment_id = environment_id + def _close_override_source(self, failure_message: str) -> None: + # A custom source can raise from close. That is logged and not raised, so the rest of + # the client still stops, or still starts after a failed start. + source = self._override_source + self._override_source = None + if source is None: + return + try: + source.close() + except Exception as e: + log.error("%s: %s", failure_message, e) + @property def environment_id(self) -> Optional[str]: """Get the environment ID reported by LaunchDarkly, if known.""" diff --git a/ldclient/impl/overrides/overlay.py b/ldclient/impl/overrides/overlay.py index 685352d6..c1fd0829 100644 --- a/ldclient/impl/overrides/overlay.py +++ b/ldclient/impl/overrides/overlay.py @@ -12,10 +12,8 @@ from ldclient.versioned_data_kind import VersionedDataKind -def _merge_all(base_items: Optional[Dict[str, Any]], override_items: Dict[str, Any]) -> Dict[str, Any]: +def _merge_all(base_items: Dict[str, Any], override_items: Dict[str, Any]) -> Dict[str, Any]: """The union of the base items and the override items. The override entry wins for a key present in both.""" - if base_items is None: - return dict(override_items) if len(override_items) == 0: return base_items merged = dict(base_items) @@ -30,9 +28,8 @@ class OverrideStoreView(ReadOnlyStore): A per-key read serves an override entry whatever the state of the base store, so an uninitialized base still serves overrides. An enumeration is the union of the base items and the layer's items, and the override entry wins for a key present in both, including a - key the base holds as a deleted item. When the base enumeration fails and the layer holds - entries, the layer's entries alone are returned. When the layer is empty, the failure is - raised as before. + key the base holds as a deleted item. A failure of the base enumeration propagates, as it + does without the overlay: the layer's entries alone would look like a complete result. """ def __init__(self, base: ReadOnlyStore, layer: OverrideLayer): @@ -46,14 +43,12 @@ def get(self, kind: VersionedDataKind, key: str, callback: Callable[[Any], Any] return self._base.get(kind, key, callback) def all(self, kind: VersionedDataKind, callback: Callable[[Any], Any] = lambda x: x) -> Any: - override_items = self._layer.all(kind) - try: - base_items = self._base.all(kind, lambda x: x) - except Exception: - if len(override_items) == 0: - raise - base_items = None - return callback(_merge_all(base_items, override_items)) + base_items = self._base.all(kind, lambda x: x) + if base_items is None: + # A store returns no result when it cannot read. That stays a failure rather than + # becoming the layer's entries alone. + return callback(None) + return callback(_merge_all(base_items, self._layer.all(kind))) @property def initialized(self) -> bool: @@ -75,11 +70,4 @@ async def get(self, kind: VersionedDataKind, key: str) -> Optional[Any]: return await self._base.get(kind, key) async def all(self, kind: VersionedDataKind) -> Dict[str, Any]: - override_items = self._layer.all(kind) - try: - base_items: Optional[Dict[str, Any]] = await self._base.all(kind) - except Exception: - if len(override_items) == 0: - raise - base_items = None - return _merge_all(base_items, override_items) + return _merge_all(await self._base.all(kind), self._layer.all(kind)) diff --git a/ldclient/impl/overrides/sink.py b/ldclient/impl/overrides/sink.py index 03866dc9..f2389628 100644 --- a/ldclient/impl/overrides/sink.py +++ b/ldclient/impl/overrides/sink.py @@ -44,21 +44,47 @@ def __init__( self._notify = notify self._has_listeners = has_listeners self._lock = threading.Lock() + self._closed = False # guarded by _lock def set_overrides(self, flags: Dict[str, Any], segments: Dict[str, Any]) -> None: # type: ignore[override] with self._lock: - if not self._has_listeners(): - self._layer.set_all(flags, segments) + if self._closed: + log.debug("Ignoring an override snapshot supplied after the override source was closed") return + self._apply(flags, segments) - previous, current = self._layer.set_all(flags, segments) - old_merged = snapshot_merged_view(self._base, previous) - new_merged = snapshot_merged_view(self._base, current) - affected = compute_affected_flags(previous, current, old_merged, new_merged) - if len(affected) > 0: - log.debug("Override update affected %d flag(s)", len(affected)) - for key in sorted(affected): - self._notify(key) + def close(self) -> None: + """ + Stops accepting snapshots. Used when the client is closed: a snapshot that the source + supplies while it shuts down is ignored. The layer keeps its contents. + """ + with self._lock: + self._closed = True + + def close_and_clear(self) -> None: + """ + Stops accepting snapshots and clears the layer. Used when the source failed to start: a + snapshot the source supplied before failing is removed, so the client runs without + overrides, and a late snapshot from the closed source is ignored. + """ + with self._lock: + self._closed = True + self._apply({}, {}) + + def _apply(self, flags: Dict[str, Any], segments: Dict[str, Any]) -> None: + # Called with _lock held. + if not self._has_listeners(): + self._layer.set_all(flags, segments) + return + + previous, current = self._layer.set_all(flags, segments) + old_merged = snapshot_merged_view(self._base, previous) + new_merged = snapshot_merged_view(self._base, current) + affected = compute_affected_flags(previous, current, old_merged, new_merged) + if len(affected) > 0: + log.debug("Override update affected %d flag(s)", len(affected)) + for key in sorted(affected): + self._notify(key) def snapshot_merged_view(base: ReadOnlyStore, overrides: LayerContents) -> MergedView: diff --git a/ldclient/interfaces.py b/ldclient/interfaces.py index 90e6e549..271969ee 100644 --- a/ldclient/interfaces.py +++ b/ldclient/interfaces.py @@ -2183,6 +2183,11 @@ def start(self, sink: OverrideSink) -> None: :meth:`close` is called. A failed load leaves the previously supplied layer untouched by not calling the sink. + An exception raised from this method does not prevent the client from starting: the SDK + logs it, closes the source, removes any snapshot the source supplied, and the client runs + without overrides. A source that cannot load its data should log that and retry rather + than raise, as the SDK's file-based source does. + :param sink: the sink that receives each snapshot """ raise NotImplementedError @@ -2190,7 +2195,9 @@ def start(self, sink: OverrideSink) -> None: @abstractmethod def close(self) -> None: """ - Stops the source and releases any resources it holds. + Stops the source and releases any resources it holds. An exception raised from this + method is logged and does not stop the client from shutting down. A snapshot that the + source supplies after the client begins to close it is ignored. """ raise NotImplementedError diff --git a/ldclient/testing/impl/test_overrides.py b/ldclient/testing/impl/test_overrides.py index 58502102..cfef2d9a 100644 --- a/ldclient/testing/impl/test_overrides.py +++ b/ldclient/testing/impl/test_overrides.py @@ -249,15 +249,30 @@ def test_overlay_get_all_with_empty_layer_is_passthrough(): overlay.all(FEATURES) -def test_overlay_get_all_serves_overrides_when_base_fails(): +def test_overlay_get_all_propagates_a_base_failure_when_the_layer_holds_entries(): + # The layer's entries alone would look like a complete result to a caller such as the + # all-flags state, so the failure propagates as it does without the overlay. A per-key read + # still serves the override. base = FakeBaseStore() base.fail_all = True layer = OverrideLayer() - layer.set_all({'override-1': flag('override-1', 1), 'override-2': flag('override-2', 2)}, {}) + layer.set_all({'override-1': flag('override-1', 1)}, {}) overlay = OverrideStoreView(base, layer) - items = overlay.all(FEATURES) - assert sorted(items.keys()) == ['override-1', 'override-2'] - assert all(item.is_override for item in items.values()) + with pytest.raises(RuntimeError): + overlay.all(FEATURES) + assert overlay.get(FEATURES, 'override-1').is_override is True + + +def test_overlay_get_all_keeps_a_missing_base_result_missing(): + # A store returns no result when it cannot read. The overlay does not turn that into the + # layer's entries alone. + class NoResultStore(FakeBaseStore): + def all(self, kind, callback=lambda x: x): + return callback(None) + + layer = OverrideLayer() + layer.set_all({'override-1': flag('override-1', 1)}, {}) + assert OverrideStoreView(NoResultStore(), layer).all(FEATURES) is None def test_overlay_initialized_follows_the_base(): @@ -295,9 +310,11 @@ async def test_async_overlay_all_when_base_fails(): overlay = AsyncOverrideStoreView(FakeAsyncBaseStore(base), layer) with pytest.raises(RuntimeError): await overlay.all(FEATURES) + # The failure propagates whatever the layer holds. layer.set_all({'override-1': flag('override-1')}, {}) - items = await overlay.all(FEATURES) - assert list(items.keys()) == ['override-1'] + with pytest.raises(RuntimeError): + await overlay.all(FEATURES) + assert (await overlay.get(FEATURES, 'override-1')).is_override is True # --------------------------------------------------------------------------- @@ -390,6 +407,38 @@ def test_sink_skips_change_computation_when_nothing_listens(): assert f.layer.get(FEATURES, 'flag1').version == 2 +def test_sink_close_keeps_the_layer_and_ignores_later_snapshots(): + f = SinkFixture(FakeBaseStore()) + f.sink.set_overrides({'flag1': flag('flag1')}, {}) + assert f.take_notified() == ['flag1'] + + f.sink.close() + assert f.layer.get(FEATURES, 'flag1') is not None + assert f.take_notified() == [] + + # A snapshot after closing changes nothing. + f.sink.set_overrides({'flag2': flag('flag2')}, {}) + assert f.layer.get(FEATURES, 'flag1') is not None + assert f.layer.get(FEATURES, 'flag2') is None + assert f.take_notified() == [] + + +def test_sink_close_and_clear_clears_the_layer_and_ignores_later_snapshots(): + f = SinkFixture(FakeBaseStore()) + f.sink.set_overrides({'flag1': flag('flag1')}, {}) + assert f.take_notified() == ['flag1'] + + # Clearing removes what the source supplied, and the removal is a change. + f.sink.close_and_clear() + assert f.layer.is_empty is True + assert f.take_notified() == ['flag1'] + + # A snapshot after closing changes nothing. + f.sink.set_overrides({'flag1': flag('flag1')}, {}) + assert f.layer.is_empty is True + assert f.take_notified() == [] + + def test_sink_notifies_directly_changed_keys_when_base_read_fails(): base = FakeBaseStore(flags={'flag1': flag('flag1')}) base.fail_all = True diff --git a/ldclient/testing/mock_components.py b/ldclient/testing/mock_components.py index f8c2e85e..a569f829 100644 --- a/ldclient/testing/mock_components.py +++ b/ldclient/testing/mock_components.py @@ -69,13 +69,24 @@ class MockOverrideSource: on every later call to set_overrides, and records its lifecycle calls. """ - def __init__(self, flags: Optional[Dict[str, Any]] = None, segments: Optional[Dict[str, Any]] = None, start_error: Optional[Exception] = None): + def __init__( + self, + flags: Optional[Dict[str, Any]] = None, + segments: Optional[Dict[str, Any]] = None, + start_error: Optional[Exception] = None, + supply_before_start_error: bool = False, + close_error: Optional[Exception] = None, + ): """ :param start_error: when given, start raises it instead of supplying the contents + :param supply_before_start_error: with start_error, start supplies the contents and then raises + :param close_error: when given, close raises it after recording the call """ self._flags: Dict[str, Any] = dict(flags or {}) self._segments: Dict[str, Any] = dict(segments or {}) self._start_error = start_error + self._supply_before_start_error = supply_before_start_error + self._close_error = close_error self._sink: Optional[OverrideSink] = None self.start_count = 0 self.close_count = 0 @@ -83,12 +94,17 @@ def __init__(self, flags: Optional[Dict[str, Any]] = None, segments: Optional[Di def start(self, sink: OverrideSink) -> None: self.start_count += 1 if self._start_error is not None: + if self._supply_before_start_error: + self._sink = sink + sink.set_overrides(self._flags, self._segments) raise self._start_error self._sink = sink sink.set_overrides(self._flags, self._segments) def close(self) -> None: self.close_count += 1 + if self._close_error is not None: + raise self._close_error def set_overrides(self, flags: Optional[Dict[str, Any]] = None, segments: Optional[Dict[str, Any]] = None) -> None: """Replaces the source's contents and, once started, pushes them to the sink.""" diff --git a/ldclient/testing/test_async_client_overrides.py b/ldclient/testing/test_async_client_overrides.py index bce576e0..c5e502e7 100644 --- a/ldclient/testing/test_async_client_overrides.py +++ b/ldclient/testing/test_async_client_overrides.py @@ -3,6 +3,7 @@ client tests. """ import asyncio +import logging import threading from typing import Any, Dict, List, Optional @@ -80,6 +81,24 @@ async def is_initialized(self) -> bool: return False +class AsyncStoreThatCannotListItems(AsyncInMemoryFeatureStore): + """An initialized persistent store whose enumeration fails.""" + + @property + def initialized(self) -> bool: + return True + + async def is_initialized(self) -> bool: + return True + + async def all(self, kind): + raise RuntimeError("store cannot list items") + + +def records_containing(caplog, level: int, text: str): + return [r for r in caplog.records if r.levelno == level and text in r.getMessage()] + + async def uninitialized_store_holding(key: str) -> UninitializedAsyncStore: store = UninitializedAsyncStore() await store.upsert(FEATURES, FlagBuilder(key).version(1).on(False).off_variation(0).variations('ld-value').build().to_json_dict()) @@ -221,16 +240,82 @@ async def test_all_flags_state_is_invalid_when_the_store_is_uninitialized_and_th @pytest.mark.asyncio -async def test_override_source_start_failure_fails_start_and_closes_the_source(): - source = MockOverrideSource(start_error=RuntimeError("cannot start")) - datasystem = AsyncDataSystemConfig(synchronizers=[MockDataSourceBuilder(AsyncHangingSynchronizer())], override_source=source.builder) +async def test_all_flags_state_before_initialization_excludes_launchdarkly_data_from_an_uninitialized_store(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source, store=await uninitialized_store_holding('ld-flag')) + try: + state = await client.all_flags_state(user) + assert state.valid is True + assert state.to_values_map() == {'overridden-flag': True} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_all_flags_state_is_invalid_when_the_store_fails_even_with_overrides(caplog): + # A daemon-mode client whose store cannot list its flags reports an invalid state and logs + # the failure, as it does without a source, rather than a valid state of the overrides alone. + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + datasystem = AsyncDataSystemConfig(override_source=source.builder, data_store=AsyncStoreThatCannotListItems(), data_store_mode=DataStoreMode.READ_ONLY) config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor()) client = AsyncLDClient(config) - with pytest.raises(RuntimeError): - await client.start(start_wait=0) + await client.start(start_wait=0) + try: + state = await client.all_flags_state(user) + assert state.valid is False + assert state.to_values_map() == {} + assert records_containing(caplog, logging.ERROR, 'Unable to read flags for all_flag_state') != [] + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_override_source_start_failure_is_logged_and_the_client_runs_without_overrides(caplog): + # The SDK's own source never raises from start. A custom source could, and that must not + # keep the client from starting: the failure is logged, the source is closed, the snapshot + # it supplied before failing is removed, and a later snapshot is ignored. + flags = {'overridden-flag': single_value_flag('overridden-flag', True)} + source = MockOverrideSource(flags=flags, start_error=RuntimeError("cannot start"), supply_before_start_error=True) + client = await make_uninitialized_client(source) + try: + assert source.close_count == 1 + assert records_containing(caplog, logging.ERROR, 'Unable to start the override source; the client runs without overrides: cannot start') != [] + detail = await client.variation_detail('overridden-flag', user, False) + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'}) + + source.set_overrides(flags) + assert await client.variation('overridden-flag', user, False) is False + finally: + await client.close() assert source.close_count == 1 +@pytest.mark.asyncio +async def test_snapshot_supplied_after_the_client_is_closed_is_ignored(): + source = MockOverrideSource() + client = await make_uninitialized_client(source) + await client.close() + + source.set_overrides({'late-flag': single_value_flag('late-flag', True)}) + + assert client._data_system.override_layer.get(FEATURES, 'late-flag') is None + + +@pytest.mark.asyncio +async def test_override_source_is_not_started_when_offline(caplog): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + datasystem = AsyncDataSystemConfig(override_source=source.builder) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, offline=True) + client = AsyncLDClient(config) + await client.start(start_wait=0) + try: + assert source.start_count == 0 + assert await client.variation('overridden-flag', user, False) is False + assert records_containing(caplog, logging.WARNING, 'Offline mode is enabled; the override source is not started and no overrides are served') != [] + finally: + await client.close() + + @pytest.mark.asyncio async def test_invalid_override_source_configuration_fails_construction(): datasystem = AsyncDataSystemConfig(synchronizers=[MockDataSourceBuilder(AsyncHangingSynchronizer())], override_source=FailingOverrideSourceBuilder()) diff --git a/ldclient/testing/test_ldclient_overrides.py b/ldclient/testing/test_ldclient_overrides.py index 95017a21..d48995c0 100644 --- a/ldclient/testing/test_ldclient_overrides.py +++ b/ldclient/testing/test_ldclient_overrides.py @@ -9,7 +9,7 @@ import pytest from ldclient.client import Config, Context, LDClient -from ldclient.datasystem import custom +from ldclient.datasystem import custom, daemon from ldclient.evaluation import EvaluationDetail from ldclient.feature_store import InMemoryFeatureStore from ldclient.impl.datasystem.fdv1 import FDv1 @@ -106,6 +106,21 @@ def uninitialized_store_holding(key: str) -> UninitializedStore: return store +class StoreThatCannotListItems(InMemoryFeatureStore): + """An initialized persistent store whose enumeration fails.""" + + @property + def initialized(self) -> bool: + return True + + def all(self, kind, callback=lambda x: x): + raise RuntimeError("store cannot list items") + + +def errors_containing(caplog, text: str): + return [r for r in caplog.records if r.levelno == logging.ERROR and text in r.getMessage()] + + def test_override_is_served_when_client_is_not_initialized(): source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) with make_uninitialized_client(source) as client: @@ -175,6 +190,37 @@ def test_all_flags_state_is_invalid_when_the_store_is_uninitialized_and_the_laye assert state.to_values_map() == {} +def test_all_flags_state_is_invalid_when_the_store_is_uninitialized_and_the_layer_holds_no_flags(): + # A layer that holds only segments has no flags to report, so the LaunchDarkly data in the + # store stays out, as it does without an override source. + source = MockOverrideSource(segments={'segment1': SegmentBuilder('segment1').version(1).build().to_json_dict()}) + with make_uninitialized_client(source, store=uninitialized_store_holding('ld-flag')) as client: + state = client.all_flags_state(user) + assert state.valid is False + assert state.to_values_map() == {} + + +def test_all_flags_state_before_initialization_excludes_launchdarkly_data_from_an_uninitialized_store(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source, store=uninitialized_store_holding('ld-flag')) as client: + state = client.all_flags_state(user) + assert state.valid is True + assert state.to_values_map() == {'overridden-flag': True} + + +def test_all_flags_state_is_invalid_when_the_store_fails_even_with_overrides(caplog): + # A daemon-mode client whose store cannot list its flags reports an invalid state and logs + # the failure, as it does without a source, rather than a valid state of the overrides alone. + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + datasystem = daemon(StoreThatCannotListItems()).overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + with LDClient(config, start_wait=0) as client: + state = client.all_flags_state(user) + assert state.valid is False + assert state.to_values_map() == {} + assert errors_containing(caplog, 'Unable to read flags for all_flag_state') != [] + + def test_override_source_is_started_before_the_constructor_returns(): source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) client = make_uninitialized_client(source) @@ -200,30 +246,52 @@ def test_invalid_override_source_configuration_fails_construction(): LDClient(config, start_wait=0) -def test_override_source_start_failure_fails_construction_and_stops_the_started_components(): - stopped = [] - - class RecordingEventProcessor(MockEventProcessor): - def stop(self): - stopped.append(True) - +def test_override_source_start_failure_is_logged_and_the_client_runs_without_overrides(caplog): + # The SDK's own source never raises from start. A custom source could, and that must not + # keep the client from starting: the failure is logged, the source is closed, and the + # client behaves as it does without overrides. source = MockOverrideSource(start_error=RuntimeError("cannot start")) - datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(source.builder).build() - config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=RecordingEventProcessor) - with pytest.raises(RuntimeError): - LDClient(config, start_wait=0) - # The source and the event processor that were set up before the failure are closed. + with make_uninitialized_client(source) as client: + assert source.close_count == 1 + assert errors_containing(caplog, 'Unable to start the override source; the client runs without overrides: cannot start') != [] + detail = client.variation_detail('any-flag', user, False) + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'}) + # Closing the client does not close the failed source again. assert source.close_count == 1 - assert stopped == [True] -def test_override_source_is_not_started_when_offline(): +def test_override_source_that_supplies_a_snapshot_and_then_fails_to_start_leaves_no_overrides(): + # The snapshot the source supplied before it failed is removed, so the client runs without + # overrides, as the log says, and a later snapshot from the closed source is ignored. + flags = {'overridden-flag': single_value_flag('overridden-flag', True)} + source = MockOverrideSource(flags=flags, start_error=RuntimeError("cannot start"), supply_before_start_error=True) + with make_uninitialized_client(source) as client: + detail = client.variation_detail('overridden-flag', user, False) + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'}) + assert client.all_flags_state(user).valid is False + + source.set_overrides(flags) + assert client.variation('overridden-flag', user, False) is False + + +def test_snapshot_supplied_after_the_client_is_closed_is_ignored(): + source = MockOverrideSource() + client = make_uninitialized_client(source) + client.close() + + source.set_overrides({'late-flag': single_value_flag('late-flag', True)}) + + assert client._data_system.override_layer.get(FEATURES, 'late-flag') is None + + +def test_override_source_is_not_started_when_offline(caplog): source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) datasystem = custom().overrides(source.builder).build() config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, offline=True) with LDClient(config, start_wait=0) as client: assert source.start_count == 0 assert client.variation('overridden-flag', user, False) is False + assert warnings_containing(caplog, 'Offline mode is enabled; the override source is not started and no overrides are served') != [] def test_data_system_without_override_source_has_no_override_layer(): From 838de6800bff4338c0ba2af5355f3fcaf30f6ee5 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:40:20 +0000 Subject: [PATCH 08/26] docs: State that an override source's start is not intended to fail A source logs and retries a failed load, and raises a configuration error from its builder's build. If start raises anyway, the SDK logs it, closes the source, removes its snapshot, and the client runs without overrides. --- ldclient/interfaces.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ldclient/interfaces.py b/ldclient/interfaces.py index 271969ee..9fec157e 100644 --- a/ldclient/interfaces.py +++ b/ldclient/interfaces.py @@ -2183,10 +2183,11 @@ def start(self, sink: OverrideSink) -> None: :meth:`close` is called. A failed load leaves the previously supplied layer untouched by not calling the sink. - An exception raised from this method does not prevent the client from starting: the SDK - logs it, closes the source, removes any snapshot the source supplied, and the client runs - without overrides. A source that cannot load its data should log that and retry rather - than raise, as the SDK's file-based source does. + This method is not intended to fail. A source that cannot load its data logs the failure + and keeps retrying, as the SDK's file-based source does. A configuration error belongs in + the builder's ``build``, where it is raised when the client is created. If this method does + raise, the client still starts: the SDK logs the exception, closes the source, removes any + snapshot the source supplied, and the client runs without overrides. :param sink: the sink that receives each snapshot """ From 5aeb04f03f95cbf3ff22ff5fef98b5bd68e8e705 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:45:08 +0000 Subject: [PATCH 09/26] fix: Keep the override flags in the all flags state when the store cannot list its flags When the store's all-items read fails, the client logs the failure as before and, if the override layer holds flags, returns a state of those flags marked invalid instead of an empty one. Overrides only add to the state, and a client-side SDK bootstrapped from it uses its flags whatever $valid says, so the overridden values still reach it. With no override flags the state is the same invalid, empty state as without a source. --- ldclient/async_client.py | 14 +++++++++-- ldclient/client.py | 14 +++++++++-- .../testing/test_async_client_overrides.py | 24 ++++++++++++++++--- ldclient/testing/test_ldclient_overrides.py | 19 ++++++++++++--- 4 files changed, 61 insertions(+), 10 deletions(-) diff --git a/ldclient/async_client.py b/ldclient/async_client.py index e4759f90..9777471d 100644 --- a/ldclient/async_client.py +++ b/ldclient/async_client.py @@ -608,7 +608,7 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState log.warning("Context was invalid for all_flags_state (%s); returning default value" % context.error) return FeatureFlagsState(False) - state = FeatureFlagsState(True) + valid = True client_only = kwargs.get('client_side_only', False) with_reasons = kwargs.get('with_reasons', False) details_only_if_tracked = kwargs.get('details_only_for_tracked_flags', False) @@ -624,8 +624,18 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState raise ValueError("feature store error") except Exception as e: log.error("Unable to read flags for all_flag_state: %s" % repr(e)) - return FeatureFlagsState(False) + # Overrides only add to the state. Without them the state is invalid and empty + # here, so it holds the override layer's flags and stays invalid. A client-side + # SDK bootstrapped from the state uses its flags whatever $valid says (the + # JavaScript client SDKs 3.x and 4.x only warn), so the overridden values still + # reach it. + layer = self._data_system.override_layer + flags_map = layer.all(FEATURES) if layer is not None else {} + if len(flags_map) == 0: + return FeatureFlagsState(False) + valid = False + state = FeatureFlagsState(valid) for key, flag in flags_map.items(): if client_only and not flag.get('clientSide', False): continue diff --git a/ldclient/client.py b/ldclient/client.py index d7796369..459fa300 100644 --- a/ldclient/client.py +++ b/ldclient/client.py @@ -598,7 +598,7 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: log.warning("Context was invalid for all_flags_state (%s); returning default value" % context.error) return FeatureFlagsState(False) - state = FeatureFlagsState(True) + valid = True client_only = kwargs.get('client_side_only', False) with_reasons = kwargs.get('with_reasons', False) details_only_if_tracked = kwargs.get('details_only_for_tracked_flags', False) @@ -616,8 +616,18 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: raise ValueError("feature store error") except Exception as e: log.error("Unable to read flags for all_flag_state: %s" % repr(e)) - return FeatureFlagsState(False) + # Overrides only add to the state. Without them the state is invalid and empty + # here, so it holds the override layer's flags and stays invalid. A client-side + # SDK bootstrapped from the state uses its flags whatever $valid says (the + # JavaScript client SDKs 3.x and 4.x only warn), so the overridden values still + # reach it. + layer = self._data_system.override_layer + flags_map = layer.all(FEATURES) if layer is not None else {} + if len(flags_map) == 0: + return FeatureFlagsState(False) + valid = False + state = FeatureFlagsState(valid) for key, flag in flags_map.items(): if client_only and not flag.get('clientSide', False): continue diff --git a/ldclient/testing/test_async_client_overrides.py b/ldclient/testing/test_async_client_overrides.py index c5e502e7..55e234f4 100644 --- a/ldclient/testing/test_async_client_overrides.py +++ b/ldclient/testing/test_async_client_overrides.py @@ -252,14 +252,32 @@ async def test_all_flags_state_before_initialization_excludes_launchdarkly_data_ @pytest.mark.asyncio -async def test_all_flags_state_is_invalid_when_the_store_fails_even_with_overrides(caplog): - # A daemon-mode client whose store cannot list its flags reports an invalid state and logs - # the failure, as it does without a source, rather than a valid state of the overrides alone. +async def test_all_flags_state_holds_the_overrides_and_is_invalid_when_the_store_fails(caplog): + # A daemon-mode client whose store cannot list its flags logs the failure, as it does without + # a source. Overrides only add to the state, so it holds the override flags, and it stays + # invalid because the store's flags are missing. source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) datasystem = AsyncDataSystemConfig(override_source=source.builder, data_store=AsyncStoreThatCannotListItems(), data_store_mode=DataStoreMode.READ_ONLY) config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor()) client = AsyncLDClient(config) await client.start(start_wait=0) + try: + state = await client.all_flags_state(user, with_reasons=True) + assert state.valid is False + assert state.to_values_map() == {'overridden-flag': True} + assert state.get_flag_reason('overridden-flag') == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + assert records_containing(caplog, logging.ERROR, 'Unable to read flags for all_flag_state') != [] + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_all_flags_state_is_invalid_and_empty_when_the_store_fails_and_the_layer_holds_no_flags(caplog): + source = MockOverrideSource() + datasystem = AsyncDataSystemConfig(override_source=source.builder, data_store=AsyncStoreThatCannotListItems(), data_store_mode=DataStoreMode.READ_ONLY) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor()) + client = AsyncLDClient(config) + await client.start(start_wait=0) try: state = await client.all_flags_state(user) assert state.valid is False diff --git a/ldclient/testing/test_ldclient_overrides.py b/ldclient/testing/test_ldclient_overrides.py index d48995c0..9917dea5 100644 --- a/ldclient/testing/test_ldclient_overrides.py +++ b/ldclient/testing/test_ldclient_overrides.py @@ -208,12 +208,25 @@ def test_all_flags_state_before_initialization_excludes_launchdarkly_data_from_a assert state.to_values_map() == {'overridden-flag': True} -def test_all_flags_state_is_invalid_when_the_store_fails_even_with_overrides(caplog): - # A daemon-mode client whose store cannot list its flags reports an invalid state and logs - # the failure, as it does without a source, rather than a valid state of the overrides alone. +def test_all_flags_state_holds_the_overrides_and_is_invalid_when_the_store_fails(caplog): + # A daemon-mode client whose store cannot list its flags logs the failure, as it does without + # a source. Overrides only add to the state, so it holds the override flags, and it stays + # invalid because the store's flags are missing. source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) datasystem = daemon(StoreThatCannotListItems()).overrides(source.builder).build() config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + with LDClient(config, start_wait=0) as client: + state = client.all_flags_state(user, with_reasons=True) + assert state.valid is False + assert state.to_values_map() == {'overridden-flag': True} + assert state.get_flag_reason('overridden-flag') == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + assert errors_containing(caplog, 'Unable to read flags for all_flag_state') != [] + + +def test_all_flags_state_is_invalid_and_empty_when_the_store_fails_and_the_layer_holds_no_flags(caplog): + source = MockOverrideSource() + datasystem = daemon(StoreThatCannotListItems()).overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) with LDClient(config, start_wait=0) as client: state = client.all_flags_state(user) assert state.valid is False From aa6274bcc0f935f8da8d01ac402211659a0a404f Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:56:58 +0000 Subject: [PATCH 10/26] fix: Mark the EXCEPTION reason of a failed evaluation of an override flag An evaluation that fails after reading a definition from the override layer is still marked (OVERRIDE 1.5.4). The single-flag and all-flags error paths of both clients now set overrideAffected on the EXCEPTION reason from the flag's own marker, read through is_override_definition so that the error path cannot fail on an unexpected store item. --- ldclient/async_client.py | 12 ++++++++-- ldclient/client.py | 11 ++++++++- ldclient/impl/evaluator_common.py | 10 ++++++++ .../testing/impl/test_evaluator_overrides.py | 11 ++++++++- .../testing/test_async_client_overrides.py | 24 +++++++++++++++++++ ldclient/testing/test_ldclient_overrides.py | 20 ++++++++++++++++ 6 files changed, 84 insertions(+), 4 deletions(-) diff --git a/ldclient/async_client.py b/ldclient/async_client.py index 9777471d..54f1ada0 100644 --- a/ldclient/async_client.py +++ b/ldclient/async_client.py @@ -28,7 +28,7 @@ ) from ldclient.impl.client_common import secure_mode_hash as _secure_mode_hash from ldclient.impl.datasystem import AsyncDataSystem, DataAvailability -from ldclient.impl.evaluator_common import error_reason +from ldclient.impl.evaluator_common import error_reason, is_override_definition from ldclient.impl.events.async_event_processor import ( DefaultAsyncEventProcessor ) @@ -553,6 +553,10 @@ def not_ready() -> Tuple[EvaluationDetail, Optional[FeatureFlag]]: log.error("Unexpected error while evaluating feature flag \"%s\": %s" % (key, repr(e))) log.debug(traceback.format_exc()) reason = error_reason('EXCEPTION') + # The evaluation read the flag definition before it failed, so a definition from + # the override layer marks the result. + if is_override_definition(flag): + reason['overrideAffected'] = True self._send_event(event_factory.new_default_event(flag, context, default, reason)) return EvaluationDetail(default, None, reason), flag @@ -646,7 +650,11 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState except Exception as e: log.error("Error evaluating flag \"%s\" in all_flags_state: %s" % (key, repr(e))) log.debug(traceback.format_exc()) - reason = {'kind': 'ERROR', 'errorKind': 'EXCEPTION'} + reason = error_reason('EXCEPTION') + # The evaluation read the flag definition before it failed, so a definition from + # the override layer marks the result. + if is_override_definition(flag): + reason['overrideAffected'] = True detail = EvaluationDetail(None, None, reason) prerequisites = [] requires_experiment_data = EventFactory.is_experiment(flag, detail.reason) diff --git a/ldclient/client.py b/ldclient/client.py index 459fa300..9ba4dd86 100644 --- a/ldclient/client.py +++ b/ldclient/client.py @@ -28,6 +28,7 @@ from ldclient.impl.datasystem import DataAvailability, DataSystem from ldclient.impl.datasystem.fdv2 import FDv2 from ldclient.impl.evaluator import Evaluator, error_reason +from ldclient.impl.evaluator_common import is_override_definition from ldclient.impl.events.diagnostics import ( _DiagnosticAccumulator, create_diagnostic_id @@ -539,6 +540,10 @@ def not_ready() -> Tuple[EvaluationDetail, Optional[FeatureFlag]]: log.error("Unexpected error while evaluating feature flag \"%s\": %s" % (key, repr(e))) log.debug(traceback.format_exc()) reason = error_reason('EXCEPTION') + # The evaluation read the flag definition before it failed, so a definition from + # the override layer marks the result. + if is_override_definition(flag): + reason['overrideAffected'] = True self._send_event(event_factory.new_default_event(flag, context, default, reason)) return EvaluationDetail(default, None, reason), flag @@ -638,7 +643,11 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: except Exception as e: log.error("Error evaluating flag \"%s\" in all_flags_state: %s" % (key, repr(e))) log.debug(traceback.format_exc()) - reason = {'kind': 'ERROR', 'errorKind': 'EXCEPTION'} + reason = error_reason('EXCEPTION') + # The evaluation read the flag definition before it failed, so a definition from + # the override layer marks the result. + if is_override_definition(flag): + reason['overrideAffected'] = True detail = EvaluationDetail(None, None, reason) prerequisites = [] diff --git a/ldclient/impl/evaluator_common.py b/ldclient/impl/evaluator_common.py index 6e0f6a0c..7385f4e8 100644 --- a/ldclient/impl/evaluator_common.py +++ b/ldclient/impl/evaluator_common.py @@ -262,6 +262,16 @@ def error_reason(error_kind: str) -> dict: return {'kind': 'ERROR', 'errorKind': error_kind} +def is_override_definition(item: Any) -> bool: + """ + Reports whether a flag or segment read from the store came from the override layer. The + data systems decode the dicts a store holds into model objects before the client reads + them, but an error path reads the marker this way so that an unexpected item cannot make + it fail too. + """ + return getattr(item, 'is_override', False) is True + + def mark_override_affected(detail: EvaluationDetail) -> None: """ Sets the override-affected indicator on the evaluation reason. The indicator is present diff --git a/ldclient/testing/impl/test_evaluator_overrides.py b/ldclient/testing/impl/test_evaluator_overrides.py index f69bc612..309c3a6b 100644 --- a/ldclient/testing/impl/test_evaluator_overrides.py +++ b/ldclient/testing/impl/test_evaluator_overrides.py @@ -9,7 +9,7 @@ import pytest from ldclient import Context -from ldclient.impl.evaluator_common import EvalResult +from ldclient.impl.evaluator_common import EvalResult, is_override_definition from ldclient.impl.events.types import EventFactory, EventInputEvaluation from ldclient.testing.builders import * from ldclient.testing.impl.evaluator_util import EvaluatorBuilder @@ -314,3 +314,12 @@ def test_reason_indicator_is_absent_when_events_do_not_carry_reasons(): flag = make_override_test_flag('feature').with_override_marker() result = EvaluatorBuilder().build().evaluate(flag, override_test_context, EventFactory(False, lambda: 0)) assert_override_affected(True, result) + + +def test_is_override_definition_reads_the_marker_and_never_fails(): + flag = FlagBuilder('feature').on(False).off_variation(0).variations('off', 'on').build() + assert is_override_definition(flag.with_override_marker()) is True + assert is_override_definition(flag) is False + # An item without the marker attribute, such as a raw dict, reads as unmarked. + assert is_override_definition(flag.to_json_dict()) is False + assert is_override_definition(None) is False diff --git a/ldclient/testing/test_async_client_overrides.py b/ldclient/testing/test_async_client_overrides.py index 55e234f4..18ed7e0b 100644 --- a/ldclient/testing/test_async_client_overrides.py +++ b/ldclient/testing/test_async_client_overrides.py @@ -415,3 +415,27 @@ async def test_flag_value_change_listener_sees_override_value_changes(): assert change.new_value == 'override-value' finally: await client.close() + + +async def raise_evaluation_failure(*args): + raise RuntimeError("evaluation failure") + + +@pytest.mark.asyncio +async def test_failed_evaluation_reason_is_marked_only_when_the_flag_came_from_the_override_layer(monkeypatch): + # The evaluation read the flag definition before it failed, so the failure of the override + # flag is marked and the failure of the ordinary flag is not, in a single evaluation and in + # the all-flags state. + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_initialized_client({'plain-flag': single_value_flag('plain-flag', True)}, source) + try: + monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure) + + assert await client.variation_detail('overridden-flag', user, 'default') == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True}) + assert await client.variation_detail('plain-flag', user, 'default') == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION'}) + + state = await client.all_flags_state(user, with_reasons=True) + assert state.get_flag_reason('overridden-flag') == {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True} + assert state.get_flag_reason('plain-flag') == {'kind': 'ERROR', 'errorKind': 'EXCEPTION'} + finally: + await client.close() diff --git a/ldclient/testing/test_ldclient_overrides.py b/ldclient/testing/test_ldclient_overrides.py index 9917dea5..792c6e0a 100644 --- a/ldclient/testing/test_ldclient_overrides.py +++ b/ldclient/testing/test_ldclient_overrides.py @@ -462,3 +462,23 @@ def test_data_source_status_is_unaffected_by_overrides(): with make_uninitialized_client(source) as client: assert client.is_initialized() is False assert client.data_source_status_provider.status.state == DataSourceState.INITIALIZING + + +def raise_evaluation_failure(*args): + raise RuntimeError("evaluation failure") + + +def test_failed_evaluation_reason_is_marked_only_when_the_flag_came_from_the_override_layer(monkeypatch): + # The evaluation read the flag definition before it failed, so the failure of the override + # flag is marked and the failure of the ordinary flag is not, in a single evaluation and in + # the all-flags state. + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_initialized_client({'plain-flag': single_value_flag('plain-flag', True)}, source) as client: + monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure) + + assert client.variation_detail('overridden-flag', user, 'default') == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True}) + assert client.variation_detail('plain-flag', user, 'default') == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION'}) + + state = client.all_flags_state(user, with_reasons=True) + assert state.get_flag_reason('overridden-flag') == {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True} + assert state.get_flag_reason('plain-flag') == {'kind': 'ERROR', 'errorKind': 'EXCEPTION'} From a5701956ca7bb55d5bee72f2a62b356ec59687d3 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:20:11 -0700 Subject: [PATCH 11/26] feat: Mark override-affected evaluations in analytics events Carries the override-affected marking on the evaluation event input, so the event processor keys on the marking alone and never reads the evaluation reason. An evaluation marked as override-affected produces no individual feature event and no debug event, even when the flag requests them, and it is counted in summary events like any other evaluation. The marking is part of the summary counter key, so override-affected and ordinary evaluations of the same flag, variation, and version accumulate into separate counters, and a counter that aggregates marked evaluations carries the overrideAffected marker, present only when true, like the unknown marker. The evaluator passes each prerequisite record's own marking to its event, so a marked prerequisite record produces no individual event while an unaffected prerequisite inside a marked evaluation is recorded as usual. The client passes the top-level marking to the evaluation event, keeps the marking on the reason when a migration evaluation replaces it with a wrong-type error, and presents an override-affected flag in the all-flags state with trackEvents and trackReason false and no debugEventsUntilDate, so a consumer bootstrapped from the state sends no individual events for it. The async client mirrors these changes. The OVERRIDE specification vector runner now also checks the marking each evaluation contributes to its summary counter. --- ldclient/async_client.py | 27 ++- ldclient/client.py | 27 ++- ldclient/impl/async_evaluator.py | 2 +- ldclient/impl/evaluator.py | 2 +- .../impl/events/event_processor_common.py | 11 +- ldclient/impl/events/event_summarizer.py | 10 +- ldclient/impl/events/types.py | 10 + .../impl/events/test_event_overrides.py | 188 ++++++++++++++++++ .../impl/events/test_event_summarizer.py | 6 +- .../testing/test_async_client_overrides.py | 33 +++ .../testing/test_ldclient_override_vectors.py | 12 +- ldclient/testing/test_ldclient_overrides.py | 176 +++++++++++++++- 12 files changed, 482 insertions(+), 22 deletions(-) create mode 100644 ldclient/testing/impl/events/test_event_overrides.py diff --git a/ldclient/async_client.py b/ldclient/async_client.py index 54f1ada0..f5008f5f 100644 --- a/ldclient/async_client.py +++ b/ldclient/async_client.py @@ -486,7 +486,12 @@ async def evaluate(): tracker = OpTracker(key, flag, context, detail, default_stage) return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': stage, 'tracker': tracker}) - detail = EvaluationDetail(default_stage.value, None, error_reason('WRONG_TYPE')) + # The type mismatch replaces the reason. The evaluation read the same definitions, + # so the new reason keeps the override-affected marking. + reason = error_reason('WRONG_TYPE') + if detail.reason.get('overrideAffected') is True: + reason['overrideAffected'] = True + detail = EvaluationDetail(default_stage.value, None, reason) tracker = OpTracker(key, flag, context, detail, default_stage) return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': default_stage, 'tracker': tracker}) @@ -547,7 +552,7 @@ def not_ready() -> Tuple[EvaluationDetail, Optional[FeatureFlag]]: detail = result.detail if detail.is_default_value(): detail = EvaluationDetail(default, None, detail.reason) - self._send_event(event_factory.new_eval_event(flag, context, detail, default)) + self._send_event(event_factory.new_eval_event(flag, context, detail, default, None, result.override_affected)) return detail, flag except Exception as e: log.error("Unexpected error while evaluating feature flag \"%s\": %s" % (key, repr(e))) @@ -647,6 +652,7 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState result = await self._evaluator.evaluate(flag, context, self._event_factory_default) detail = result.detail prerequisites = result.prerequisites + override_affected = result.override_affected except Exception as e: log.error("Error evaluating flag \"%s\" in all_flags_state: %s" % (key, repr(e))) log.debug(traceback.format_exc()) @@ -657,7 +663,18 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState reason['overrideAffected'] = True detail = EvaluationDetail(None, None, reason) prerequisites = [] + override_affected = False requires_experiment_data = EventFactory.is_experiment(flag, detail.reason) + track_events = flag.get('trackEvents', False) or requires_experiment_data + track_reason = requires_experiment_data + debug_events_until_date = flag.get('debugEventsUntilDate', None) + if override_affected: + # A consumer of this state sends individual events according to these fields. + # An override-affected evaluation produces no individual events, so the state + # turns them off for this flag. The flag, its value, and its reason stay. + track_events = False + track_reason = False + debug_events_until_date = None flag_state = { 'key': flag['key'], 'value': detail.value, @@ -665,9 +682,9 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState 'reason': detail.reason, 'version': flag['version'], 'prerequisites': prerequisites, - 'trackEvents': flag.get('trackEvents', False) or requires_experiment_data, - 'trackReason': requires_experiment_data, - 'debugEventsUntilDate': flag.get('debugEventsUntilDate', None), + 'trackEvents': track_events, + 'trackReason': track_reason, + 'debugEventsUntilDate': debug_events_until_date, } state.add_flag(flag_state, with_reasons, details_only_if_tracked) diff --git a/ldclient/client.py b/ldclient/client.py index 9ba4dd86..d26ba507 100644 --- a/ldclient/client.py +++ b/ldclient/client.py @@ -471,7 +471,12 @@ def evaluate(): tracker = OpTracker(key, flag, context, detail, default_stage) return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': stage, 'tracker': tracker}) - detail = EvaluationDetail(default_stage.value, None, error_reason('WRONG_TYPE')) + # The type mismatch replaces the reason. The evaluation read the same definitions, + # so the new reason keeps the override-affected marking. + reason = error_reason('WRONG_TYPE') + if detail.reason.get('overrideAffected') is True: + reason['overrideAffected'] = True + detail = EvaluationDetail(default_stage.value, None, reason) tracker = OpTracker(key, flag, context, detail, default_stage) return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': default_stage, 'tracker': tracker}) @@ -534,7 +539,7 @@ def not_ready() -> Tuple[EvaluationDetail, Optional[FeatureFlag]]: detail = result.detail if detail.is_default_value(): detail = EvaluationDetail(default, None, detail.reason) - self._send_event(event_factory.new_eval_event(flag, context, detail, default)) + self._send_event(event_factory.new_eval_event(flag, context, detail, default, None, result.override_affected)) return detail, flag except Exception as e: log.error("Unexpected error while evaluating feature flag \"%s\": %s" % (key, repr(e))) @@ -640,6 +645,7 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: result = self._evaluator.evaluate(flag, context, self._event_factory_default) detail = result.detail prerequisites = result.prerequisites + override_affected = result.override_affected except Exception as e: log.error("Error evaluating flag \"%s\" in all_flags_state: %s" % (key, repr(e))) log.debug(traceback.format_exc()) @@ -650,8 +656,19 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: reason['overrideAffected'] = True detail = EvaluationDetail(None, None, reason) prerequisites = [] + override_affected = False requires_experiment_data = EventFactory.is_experiment(flag, detail.reason) + track_events = flag.get('trackEvents', False) or requires_experiment_data + track_reason = requires_experiment_data + debug_events_until_date = flag.get('debugEventsUntilDate', None) + if override_affected: + # A consumer of this state sends individual events according to these fields. + # An override-affected evaluation produces no individual events, so the state + # turns them off for this flag. The flag, its value, and its reason stay. + track_events = False + track_reason = False + debug_events_until_date = None flag_state = { 'key': flag['key'], 'value': detail.value, @@ -659,9 +676,9 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: 'reason': detail.reason, 'version': flag['version'], 'prerequisites': prerequisites, - 'trackEvents': flag.get('trackEvents', False) or requires_experiment_data, - 'trackReason': requires_experiment_data, - 'debugEventsUntilDate': flag.get('debugEventsUntilDate', None), + 'trackEvents': track_events, + 'trackReason': track_reason, + 'debugEventsUntilDate': debug_events_until_date, } state.add_flag(flag_state, with_reasons, details_only_if_tracked) diff --git a/ldclient/impl/async_evaluator.py b/ldclient/impl/async_evaluator.py index 935fc84d..31f3466c 100644 --- a/ldclient/impl/async_evaluator.py +++ b/ldclient/impl/async_evaluator.py @@ -148,7 +148,7 @@ async def _check_prerequisites(self, flag: FeatureFlag, context: Context, state: # off variation was. But we still need to evaluate it in order to generate an event. if (not prereq_flag.on) or prereq_res.variation_index != prereq.variation: failed_prereq = prereq - event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag) + event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag, prereq_override_affected) state.add_event(event) if failed_prereq: return {'kind': 'PREREQUISITE_FAILED', 'prerequisiteKey': failed_prereq.key} diff --git a/ldclient/impl/evaluator.py b/ldclient/impl/evaluator.py index f6a14a25..bbce2260 100644 --- a/ldclient/impl/evaluator.py +++ b/ldclient/impl/evaluator.py @@ -146,7 +146,7 @@ def _check_prerequisites(self, flag: FeatureFlag, context: Context, state: EvalR # off variation was. But we still need to evaluate it in order to generate an event. if (not prereq_flag.on) or prereq_res.variation_index != prereq.variation: failed_prereq = prereq - event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag) + event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag, prereq_override_affected) state.add_event(event) if failed_prereq: return {'kind': 'PREREQUISITE_FAILED', 'prerequisiteKey': failed_prereq.key} diff --git a/ldclient/impl/events/event_processor_common.py b/ldclient/impl/events/event_processor_common.py index 95ea278f..75f634b1 100644 --- a/ldclient/impl/events/event_processor_common.py +++ b/ldclient/impl/events/event_processor_common.py @@ -181,7 +181,7 @@ def make_summary_event(self, summary: EventSummary): } counters = [] for ckey, cval in flag_data.counters.items(): - variation, version = ckey + variation, version, override_affected = ckey counter = {'count': cval.count, 'value': cval.value} if variation is not None: counter['variation'] = variation @@ -189,6 +189,9 @@ def make_summary_event(self, summary: EventSummary): counter['unknown'] = True else: counter['version'] = version + # The marker is present only when true, like the unknown marker. + if override_affected: + counter['overrideAffected'] = True counters.append(counter) flag_data_out['counters'] = counters flags_out[key] = flag_data_out @@ -266,9 +269,11 @@ def _process_event(self, event: EventInput): context = event.context if not event.exclude_from_summaries: self._outbox.add_to_summary(event) - if event.track_events: + # An override-affected evaluation appears only in the summary counters. It produces + # no individual feature event and no debug event, even when the flag requests them. + if event.track_events and not event.override_affected: full_event = event - if self._should_debug_event(event): + if not event.override_affected and self._should_debug_event(event): debug_event = DebugEvent(event) elif isinstance(event, EventInputIdentify): if self._omit_anonymous_contexts: diff --git a/ldclient/impl/events/event_summarizer.py b/ldclient/impl/events/event_summarizer.py index acf3a88f..ca2b5b5e 100644 --- a/ldclient/impl/events/event_summarizer.py +++ b/ldclient/impl/events/event_summarizer.py @@ -22,10 +22,16 @@ def __repr__(self) -> str: # used only in test debugging return "EventSummaryCounter(%d, %s)" % (self.count, self.value) +# A counter is keyed by variation index, flag version, and whether the evaluations it counts +# were override-affected. The marker is part of the key, so override-affected and ordinary +# evaluations of the same flag, variation, and version accumulate into separate counters. +CounterKey = Tuple[Optional[int], Optional[int], bool] + + class EventSummaryFlag: __slots__ = ['context_kinds', 'default', 'counters'] - def __init__(self, context_kinds: Set[str], default: Any, counters: Dict[Tuple[Optional[int], Optional[int]], EventSummaryCounter]): + def __init__(self, context_kinds: Set[str], default: Any, counters: Dict[CounterKey, EventSummaryCounter]): self.context_kinds = context_kinds self.counters = counters self.default = default @@ -71,7 +77,7 @@ def summarize_event(self, event: EventInputEvaluation): if c is not None: flag_data.context_kinds.add(c.kind) - counter_key = (event.variation, None if event.flag is None else event.flag.version) + counter_key = (event.variation, None if event.flag is None else event.flag.version, event.override_affected) counter = flag_data.counters.get(counter_key) if counter is None: counter = EventSummaryCounter(1, event.value) diff --git a/ldclient/impl/events/types.py b/ldclient/impl/events/types.py index d0883c47..b107a3b7 100644 --- a/ldclient/impl/events/types.py +++ b/ldclient/impl/events/types.py @@ -52,6 +52,7 @@ class EventInputEvaluation(EventInput): "track_events", "sampling_ratio", "exclude_from_summaries", + "override_affected", ] def __init__( @@ -66,6 +67,7 @@ def __init__( default_value: Any, prereq_of: Optional[FeatureFlag] = None, track_events: bool = False, + override_affected: bool = False, ): super().__init__(timestamp, context, 1 if flag is None else flag.sampling_ratio) self.key = key @@ -79,6 +81,11 @@ def __init__( self.exclude_from_summaries = ( False if flag is None else flag.exclude_from_summaries ) + # True when a flag override affected this evaluation, directly or through a + # prerequisite or segment. The event processor keys on this scalar alone, never on the + # reason: such an evaluation produces no individual feature or debug event and is + # counted in a summary counter that carries the override-affected marker. + self.override_affected = override_affected def to_debugging_dict(self) -> dict: return { @@ -94,6 +101,7 @@ def to_debugging_dict(self) -> dict: "track_events": self.track_events, "exclude_from_summaries": self.exclude_from_summaries, "sampling_ratio": self.sampling_ratio, + "override_affected": self.override_affected, } @@ -155,6 +163,7 @@ def new_eval_event( detail: EvaluationDetail, default_value: Any, prereq_of_flag: Optional[FeatureFlag] = None, + override_affected: bool = False, ) -> EventInputEvaluation: add_experiment_data = self.is_experiment(flag, detail.reason) return EventInputEvaluation( @@ -168,6 +177,7 @@ def new_eval_event( default_value, prereq_of_flag, flag.track_events or add_experiment_data, + override_affected, ) def new_default_event( diff --git a/ldclient/testing/impl/events/test_event_overrides.py b/ldclient/testing/impl/events/test_event_overrides.py new file mode 100644 index 00000000..4f4345a3 --- /dev/null +++ b/ldclient/testing/impl/events/test_event_overrides.py @@ -0,0 +1,188 @@ +""" +Tests for how analytics events treat override-affected evaluations: they appear in summary +counters only, under a counter that carries the override-affected marker, and they produce no +individual feature event and no debug event. +""" +import json +import time +from typing import Any, Dict, List + +import pytest + +from ldclient.async_config import AsyncConfig +from ldclient.config import Config +from ldclient.context import Context +from ldclient.impl.events.async_event_processor import ( + DefaultAsyncEventProcessor +) +from ldclient.impl.events.event_processor import DefaultEventProcessor +from ldclient.impl.events.event_processor_common import EventOutputFormatter +from ldclient.impl.events.event_summarizer import ( + EventSummarizer, + EventSummaryCounter +) +from ldclient.impl.events.types import EventInputEvaluation +from ldclient.testing.builders import FlagBuilder +from ldclient.testing.impl.events.test_async_event_processor import MockAioHttp +from ldclient.testing.stub_util import MockHttp + +context = Context.builder('userkey').name('Red').build() +timestamp = 10000 + + +def tracked_flag(key: str = 'flagkey', version: int = 2): + """A flag that requests individual feature events and debug events far into the future.""" + return FlagBuilder(key).version(version).track_events(True).debug_events_until_date(int(time.time() * 1000) + 100000).build() + + +def evaluation(flag, value: str = 'value', variation: int = 1, override_affected: bool = False, track_events: bool = True) -> EventInputEvaluation: + return EventInputEvaluation(timestamp, context, flag.key, flag, variation, value, None, 'default', None, track_events, override_affected) + + +def events_of_kind(output: List[Dict[str, Any]], kind: str) -> List[Dict[str, Any]]: + return [e for e in output if e['kind'] == kind] + + +# --------------------------------------------------------------------------- +# Summarizer and output formatter +# --------------------------------------------------------------------------- + +def test_summarizer_keeps_override_affected_counters_separate(): + flag = tracked_flag() + es = EventSummarizer() + es.summarize_event(evaluation(flag)) + es.summarize_event(evaluation(flag, override_affected=True)) + es.summarize_event(evaluation(flag, override_affected=True)) + es.summarize_event(evaluation(flag)) + counters = es.snapshot().flags[flag.key].counters + assert counters == { + (1, flag.version, False): EventSummaryCounter(2, 'value'), + (1, flag.version, True): EventSummaryCounter(2, 'value'), + } + + +def test_summary_output_carries_the_marker_only_on_override_affected_counters(): + flag = tracked_flag() + es = EventSummarizer() + es.summarize_event(evaluation(flag)) + es.summarize_event(evaluation(flag, override_affected=True)) + es.summarize_event(EventInputEvaluation(timestamp, context, 'unknown-flag', None, None, 'default', None, 'default', None, False, False)) + output = EventOutputFormatter(Config('SDK_KEY')).make_summary_event(es.snapshot()) + + counters = output['features'][flag.key]['counters'] + assert len(counters) == 2 + by_marker = {c.get('overrideAffected'): c for c in counters} + assert by_marker[None] == {'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version} + assert by_marker[True] == {'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version, 'overrideAffected': True} + assert 'overrideAffected' not in json.dumps(output['features']['unknown-flag']) + + +def test_event_input_evaluation_defaults_to_not_override_affected(): + event = EventInputEvaluation(timestamp, context, 'flag', None, None, 'default', None, 'default') + assert event.override_affected is False + assert event.to_debugging_dict()['override_affected'] is False + + +# --------------------------------------------------------------------------- +# Sync event processor +# --------------------------------------------------------------------------- + +def flush_and_get_events(ep: DefaultEventProcessor, mock_http: MockHttp) -> List[Dict[str, Any]]: + ep.flush() + ep._wait_until_inactive() + assert mock_http.request_data is not None, 'Expected to get an HTTP request but did not get one' + return json.loads(mock_http.request_data) + + +def make_processor(mock_http: MockHttp) -> DefaultEventProcessor: + return DefaultEventProcessor(Config('SDK_KEY', diagnostic_opt_out=True), mock_http) + + +def test_override_affected_evaluation_produces_no_feature_or_debug_event(): + flag = tracked_flag() + mock_http = MockHttp() + with make_processor(mock_http) as ep: + ep.send_event(evaluation(flag, override_affected=True)) + output = flush_and_get_events(ep, mock_http) + + assert [e['kind'] for e in output] == ['index', 'summary'] + counters = output[1]['features'][flag.key]['counters'] + assert counters == [{'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version, 'overrideAffected': True}] + + +def test_ordinary_evaluation_still_produces_feature_and_debug_events(): + flag = tracked_flag() + mock_http = MockHttp() + with make_processor(mock_http) as ep: + ep.send_event(evaluation(flag)) + output = flush_and_get_events(ep, mock_http) + + assert sorted(e['kind'] for e in output) == ['debug', 'feature', 'index', 'summary'] + counters = events_of_kind(output, 'summary')[0]['features'][flag.key]['counters'] + assert counters == [{'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version}] + + +def test_mixed_evaluations_of_one_flag_split_into_two_counters_and_one_feature_event(): + flag = tracked_flag() + mock_http = MockHttp() + with make_processor(mock_http) as ep: + ep.send_event(evaluation(flag, override_affected=True)) + ep.send_event(evaluation(flag, override_affected=True)) + ep.send_event(evaluation(flag)) + output = flush_and_get_events(ep, mock_http) + + assert len(events_of_kind(output, 'feature')) == 1 + assert len(events_of_kind(output, 'debug')) == 1 + counters = events_of_kind(output, 'summary')[0]['features'][flag.key]['counters'] + assert sorted(counters, key=lambda c: c['count']) == [ + {'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version}, + {'count': 2, 'value': 'value', 'variation': 1, 'version': flag.version, 'overrideAffected': True}, + ] + + +def test_override_affected_prerequisite_record_produces_no_feature_event(): + parent = tracked_flag('parent', 1) + prereq = tracked_flag('prereq', 7) + mock_http = MockHttp() + with make_processor(mock_http) as ep: + ep.send_event(EventInputEvaluation(timestamp, context, prereq.key, prereq, 1, 'value', None, None, parent, True, True)) + output = flush_and_get_events(ep, mock_http) + + assert [e['kind'] for e in output] == ['index', 'summary'] + counters = output[1]['features'][prereq.key]['counters'] + assert counters == [{'count': 1, 'value': 'value', 'variation': 1, 'version': prereq.version, 'overrideAffected': True}] + + +def test_override_affected_evaluation_still_produces_an_index_event(): + flag = tracked_flag() + mock_http = MockHttp() + with make_processor(mock_http) as ep: + ep.send_event(evaluation(flag, override_affected=True)) + output = flush_and_get_events(ep, mock_http) + assert output[0]['kind'] == 'index' + assert output[0]['context'] == context.to_dict() + + +# --------------------------------------------------------------------------- +# Async event processor (shares the dispatch logic) +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_async_processor_suppresses_individual_events_for_override_affected_evaluations(): + flag = tracked_flag() + mock_http = MockAioHttp() + ep = DefaultAsyncEventProcessor(AsyncConfig('SDK_KEY', diagnostic_opt_out=True), mock_http) + try: + ep.send_event(evaluation(flag, override_affected=True)) + ep.send_event(evaluation(flag)) + assert await ep.flush_and_wait(5) is True + output = json.loads(mock_http.request_data) + finally: + await ep.stop() + + assert sorted(e['kind'] for e in output) == ['debug', 'feature', 'index', 'summary'] + counters = events_of_kind(output, 'summary')[0]['features'][flag.key]['counters'] + assert sorted(counters, key=lambda c: 'overrideAffected' in c) == [ + {'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version}, + {'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version, 'overrideAffected': True}, + ] diff --git a/ldclient/testing/impl/events/test_event_summarizer.py b/ldclient/testing/impl/events/test_event_summarizer.py index d7cf4ad1..3b2a2f82 100644 --- a/ldclient/testing/impl/events/test_event_summarizer.py +++ b/ldclient/testing/impl/events/test_event_summarizer.py @@ -41,8 +41,8 @@ def test_summarize_event_increments_counters(): data = es.snapshot() expected = { - 'flag1': EventSummaryFlag({'user'}, 'default1', {(1, flag1.version): EventSummaryCounter(2, 'value1'), (2, flag1.version): EventSummaryCounter(1, 'value2')}), - 'flag2': EventSummaryFlag({'user'}, 'default2', {(1, flag2.version): EventSummaryCounter(1, 'value99')}), - 'badkey': EventSummaryFlag({'user'}, 'default3', {(None, None): EventSummaryCounter(1, 'default3')}), + 'flag1': EventSummaryFlag({'user'}, 'default1', {(1, flag1.version, False): EventSummaryCounter(2, 'value1'), (2, flag1.version, False): EventSummaryCounter(1, 'value2')}), + 'flag2': EventSummaryFlag({'user'}, 'default2', {(1, flag2.version, False): EventSummaryCounter(1, 'value99')}), + 'badkey': EventSummaryFlag({'user'}, 'default3', {(None, None, False): EventSummaryCounter(1, 'default3')}), } assert data.flags == expected diff --git a/ldclient/testing/test_async_client_overrides.py b/ldclient/testing/test_async_client_overrides.py index 18ed7e0b..3556d9bf 100644 --- a/ldclient/testing/test_async_client_overrides.py +++ b/ldclient/testing/test_async_client_overrides.py @@ -5,6 +5,7 @@ import asyncio import logging import threading +import time from typing import Any, Dict, List, Optional import pytest @@ -15,6 +16,7 @@ from ldclient.context import Context from ldclient.evaluation import EvaluationDetail from ldclient.impl.aio.concurrency import AsyncEvent +from ldclient.impl.events.types import EventInputEvaluation from ldclient.impl.integrations.files.filedata import make_flag_with_value from ldclient.interfaces import AsyncFeatureStore, DataStoreMode from ldclient.testing.builders import FlagBuilder @@ -439,3 +441,34 @@ async def test_failed_evaluation_reason_is_marked_only_when_the_flag_came_from_t assert state.get_flag_reason('plain-flag') == {'kind': 'ERROR', 'errorKind': 'EXCEPTION'} finally: await client.close() + + +@pytest.mark.asyncio +async def test_override_evaluation_events_carry_override_affected_marking(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source) + try: + assert await client.variation('overridden-flag', user, False) is True + records = [e for e in client._event_processor.events if isinstance(e, EventInputEvaluation)] + assert [e.key for e in records] == ['overridden-flag'] + assert records[0].override_affected is True + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_all_flags_state_turns_off_event_tracking_for_override_affected_flags(): + overridden = FlagBuilder('overridden-flag').version(7).on(False).off_variation(0).variations(True).track_events(True).debug_events_until_date(int(time.time() * 1000) + 100000).build().to_json_dict() + plain = FlagBuilder('plain-tracked').version(1).on(False).off_variation(0).variations(True).track_events(True).build().to_json_dict() + source = MockOverrideSource(flags={'overridden-flag': overridden}) + client = await make_initialized_client({'plain-tracked': plain}, source) + try: + state = await client.all_flags_state(user, with_reasons=True) + flags_state = state.to_json_dict()['$flagsState'] + assert flags_state['plain-tracked']['trackEvents'] is True + assert 'trackEvents' not in flags_state['overridden-flag'] + assert 'debugEventsUntilDate' not in flags_state['overridden-flag'] + assert flags_state['overridden-flag']['reason']['overrideAffected'] is True + assert state.get_flag_value('overridden-flag') is True + finally: + await client.close() diff --git a/ldclient/testing/test_ldclient_override_vectors.py b/ldclient/testing/test_ldclient_override_vectors.py index d9533c3d..a55ea87d 100644 --- a/ldclient/testing/test_ldclient_override_vectors.py +++ b/ldclient/testing/test_ldclient_override_vectors.py @@ -1,7 +1,8 @@ """ Runs the OVERRIDE specification test vectors. Each vector sets up LaunchDarkly data, an override layer, and an initialization state. The test evaluates one flag through the full -client stack and checks the value, the variation index, and the reason. +client stack and checks the value, the variation index, the reason, and the marking handed to the +event processor. """ import json import os @@ -11,6 +12,7 @@ from ldclient.client import Config, Context, LDClient from ldclient.datasystem import custom +from ldclient.impl.events.types import EventInputEvaluation from ldclient.impl.integrations.files.filedata import make_flag_with_value from ldclient.testing.mock_components import ( HangingSynchronizer, @@ -83,3 +85,11 @@ def test_override_spec_vector(vector: Dict[str, Any]): assert detail.value == expect['value'], "value" assert detail.variation_index == expect['variationIndex'], "variationIndex" assert_reason(expect['reason'], detail.reason) + + # summaryOverrideAffected is the marking the client hands to the event processor for this + # evaluation. The event processor keys individual-event suppression and the summary + # counter marker on that scalar, not on the reason. + if 'summaryOverrideAffected' in expect: + records = [e for e in client._event_processor._events if isinstance(e, EventInputEvaluation) and e.key == evaluate['flagKey']] + assert len(records) == 1, "expected exactly one evaluation record for the flag" + assert records[0].override_affected is expect['summaryOverrideAffected'], "summaryOverrideAffected" diff --git a/ldclient/testing/test_ldclient_overrides.py b/ldclient/testing/test_ldclient_overrides.py index 792c6e0a..3f68134e 100644 --- a/ldclient/testing/test_ldclient_overrides.py +++ b/ldclient/testing/test_ldclient_overrides.py @@ -2,7 +2,9 @@ Tests for flag overrides through the client: the override source lifecycle, the overlay at the store read boundary, the not-initialized gate, the all-flags state, and flag change notifications. """ +import json import logging +import time from queue import Empty, Queue from typing import Any, Dict, List, Optional @@ -12,7 +14,10 @@ from ldclient.datasystem import custom, daemon from ldclient.evaluation import EvaluationDetail from ldclient.feature_store import InMemoryFeatureStore +from ldclient.hook import EvaluationSeriesContext, Hook, Metadata from ldclient.impl.datasystem.fdv1 import FDv1 +from ldclient.impl.events.event_processor import DefaultEventProcessor +from ldclient.impl.events.types import EventInputEvaluation from ldclient.impl.integrations.files.filedata import make_flag_with_value from ldclient.integrations.test_datav2 import TestDataV2 from ldclient.interfaces import ( @@ -21,6 +26,7 @@ FeatureStore, FlagChange ) +from ldclient.migrations import Stage from ldclient.testing.builders import ( FlagBuilder, FlagRuleBuilder, @@ -33,7 +39,7 @@ MockOverrideSource, StaticInitializer ) -from ldclient.testing.stub_util import MockEventProcessor +from ldclient.testing.stub_util import MockEventProcessor, MockHttp from ldclient.versioned_data_kind import FEATURES user = Context.create('user-key') @@ -482,3 +488,171 @@ def test_failed_evaluation_reason_is_marked_only_when_the_flag_came_from_the_ove state = client.all_flags_state(user, with_reasons=True) assert state.get_flag_reason('overridden-flag') == {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True} assert state.get_flag_reason('plain-flag') == {'kind': 'ERROR', 'errorKind': 'EXCEPTION'} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +def evaluation_events_by_key(client: LDClient) -> Dict[str, EventInputEvaluation]: + """The evaluation records the client handed to the event processor, keyed by flag key.""" + records = {} + processor: Any = client._event_processor + for event in processor._events: + if isinstance(event, EventInputEvaluation): + records[event.key] = event + return records + + +def test_override_evaluation_events_carry_override_affected_marking(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + assert client.variation('overridden-flag', user, False) is True + records = evaluation_events_by_key(client) + assert list(records.keys()) == ['overridden-flag'] + assert records['overridden-flag'].override_affected is True + + +def test_ordinary_evaluation_events_are_not_marked(): + normal = FlagBuilder('flag-normal').version(100).on(False).off_variation(0).variations('normal-value').track_events(True).build().to_json_dict() + source = MockOverrideSource(flags={'other': single_value_flag('other', True)}) + with make_initialized_client({'flag-normal': normal}, source) as client: + client.variation('flag-normal', user, 'default') + records = evaluation_events_by_key(client) + assert records['flag-normal'].override_affected is False + assert records['flag-normal'].track_events is True + + +def tracked_bool_flag(key: str) -> FlagBuilder: + return FlagBuilder(key).version(100).variations(False, True).off_variation(0).fallthrough_variation(1).track_events(True) + + +def test_overridden_prerequisite_marks_the_dependent_evaluation_records(): + # top-flag (LaunchDarkly) --> mid-flag (LaunchDarkly) --> leaf-flag (overridden) + # --> plain-flag (LaunchDarkly) + # The LaunchDarkly copy of leaf-flag is off, so the chain passes only through the override. + ld_data = { + 'top-flag': tracked_bool_flag('top-flag').on(True).prerequisite('mid-flag', 1).prerequisite('plain-flag', 1).build().to_json_dict(), + 'mid-flag': tracked_bool_flag('mid-flag').on(True).prerequisite('leaf-flag', 1).build().to_json_dict(), + 'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict(), + 'leaf-flag': tracked_bool_flag('leaf-flag').on(False).build().to_json_dict(), + } + source = MockOverrideSource(flags={'leaf-flag': tracked_bool_flag('leaf-flag').on(True).build().to_json_dict()}) + with make_initialized_client(ld_data, source) as client: + detail = client.variation_detail('top-flag', user, False) + assert detail.value is True + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + + records = evaluation_events_by_key(client) + assert sorted(records.keys()) == ['leaf-flag', 'mid-flag', 'plain-flag', 'top-flag'] + assert records['top-flag'].override_affected is True + assert records['mid-flag'].override_affected is True + assert records['leaf-flag'].override_affected is True + assert records['plain-flag'].override_affected is False + assert records['mid-flag'].prereq_of is not None and records['mid-flag'].prereq_of.key == 'top-flag' + assert records['leaf-flag'].prereq_of is not None and records['leaf-flag'].prereq_of.key == 'mid-flag' + assert records['leaf-flag'].reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + assert records['plain-flag'].reason == {'kind': 'FALLTHROUGH'} + + +def test_all_flags_state_turns_off_event_tracking_for_override_affected_flags(): + debug_until = int(time.time() * 1000) + 100000 + ld_data = { + 'plain-tracked': FlagBuilder('plain-tracked').version(1).on(False).off_variation(0).variations(True).track_events(True).debug_events_until_date(debug_until).build().to_json_dict(), + 'dependent-tracked': FlagBuilder('dependent-tracked').version(1).on(True).variations(False, True).fallthrough_variation(1).prerequisite('overridden-flag', 0).track_events(True).debug_events_until_date(debug_until).build().to_json_dict(), + } + # The overridden flag is on and serves variation 0, so the dependent flag's prerequisite passes. + overridden = FlagBuilder('overridden-flag').version(7).on(True).fallthrough_variation(0).variations(True).track_events(True).debug_events_until_date(debug_until).build().to_json_dict() + source = MockOverrideSource(flags={'overridden-flag': overridden}) + with make_initialized_client(ld_data, source) as client: + state = client.all_flags_state(user, with_reasons=True) + assert state.valid is True + flags_state = state.to_json_dict()['$flagsState'] + + # A flag with no override keeps its tracking fields. + assert flags_state['plain-tracked']['trackEvents'] is True + assert flags_state['plain-tracked']['debugEventsUntilDate'] == debug_until + + # The overridden flag and the flag that depends on it stay in the state with their values + # and marked reasons, but with no tracking fields. + for key in ('overridden-flag', 'dependent-tracked'): + assert flags_state[key]['reason']['overrideAffected'] is True, key + assert 'trackEvents' not in flags_state[key], key + assert 'trackReason' not in flags_state[key], key + assert 'debugEventsUntilDate' not in flags_state[key], key + assert state.get_flag_value('overridden-flag') is True + assert state.get_flag_value('dependent-tracked') is True + assert flags_state['overridden-flag']['version'] == 7 + + +def test_all_flags_state_keeps_details_of_override_affected_flags_when_details_only_for_tracked_flags(): + # With details only for tracked flags, an override-affected flag counts as untracked, so its + # version and reason are omitted like any other untracked flag, and its value stays. + overridden = FlagBuilder('overridden-flag').version(7).on(False).off_variation(0).variations(True).track_events(True).build().to_json_dict() + source = MockOverrideSource(flags={'overridden-flag': overridden}) + with make_initialized_client({}, source) as client: + state = client.all_flags_state(user, with_reasons=True, details_only_for_tracked_flags=True) + flags_state = state.to_json_dict()['$flagsState'] + assert state.get_flag_value('overridden-flag') is True + assert 'version' not in flags_state['overridden-flag'] + assert 'reason' not in flags_state['overridden-flag'] + + +def test_wrong_type_result_of_overridden_flag_stays_marked(): + details = [] + + class CapturingHook(Hook): + @property + def metadata(self) -> Metadata: + return Metadata(name='capturing-hook') + + def before_evaluation(self, series_context: EvaluationSeriesContext, data: dict) -> dict: + return data + + def after_evaluation(self, series_context: EvaluationSeriesContext, data: dict, detail: EvaluationDetail) -> dict: + details.append(detail) + return data + + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', 'not-a-stage')}) + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor, hooks=[CapturingHook()]) + with LDClient(config, start_wait=0) as client: + stage, _ = client.migration_variation('overridden-flag', user, Stage.OFF) + assert stage == Stage.OFF + assert len(details) == 1 + assert details[0].value == 'off' + assert details[0].reason == {'kind': 'ERROR', 'errorKind': 'WRONG_TYPE', 'overrideAffected': True} + + +def test_override_affected_evaluations_appear_only_in_summary_output(): + # End to end through the real event processor: the overridden flag requests individual + # feature events and debug events, and an ordinary flag requests feature events. + debug_until = int(time.time() * 1000) + 100000 + overridden = FlagBuilder('flag-tracked-override').version(300).on(False).off_variation(0).variations('override-value').track_events(True).debug_events_until_date(debug_until).build().to_json_dict() + normal = FlagBuilder('flag-normal').version(100).on(False).off_variation(0).variations('normal-value').track_events(True).build().to_json_dict() + source = MockOverrideSource(flags={'flag-tracked-override': overridden}) + mock_http = MockHttp() + + initializer = StaticInitializer({'flag-normal': normal}, {}) + datasystem = custom().initializers([initializer.builder]).overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, diagnostic_opt_out=True, event_processor_class=lambda config: DefaultEventProcessor(config, mock_http)) + with LDClient(config, start_wait=5) as client: + assert client.is_initialized() is True + for _ in range(2): + assert client.variation('flag-tracked-override', user, 'default1') == 'override-value' + assert client.variation('flag-normal', user, 'default2') == 'normal-value' + client.flush() + client._event_processor._wait_until_inactive() + + assert mock_http.request_data is not None + output = json.loads(mock_http.request_data) + kinds = sorted(e['kind'] for e in output) + assert kinds == ['feature', 'index', 'summary'] + feature = [e for e in output if e['kind'] == 'feature'][0] + assert feature['key'] == 'flag-normal' + summary = [e for e in output if e['kind'] == 'summary'][0] + assert summary['features']['flag-tracked-override']['default'] == 'default1' + assert summary['features']['flag-tracked-override']['counters'] == [ + {'count': 2, 'value': 'override-value', 'variation': 0, 'version': 300, 'overrideAffected': True} + ] + assert summary['features']['flag-normal']['counters'] == [{'count': 1, 'value': 'normal-value', 'variation': 0, 'version': 100}] From 1861888c0420c554256b268abfec4d3532171734 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Sat, 3 Oct 2026 00:51:12 +0000 Subject: [PATCH 12/26] fix: Mark a failed evaluation of an override flag and drop notifications after the loop closes When an evaluation raises and the flag definition came from the override layer, the EXCEPTION reason carries the override indicator, the default event record is marked so no individual event is produced, and the all-flags state presents the flag with tracking off. The async data system drops a flag change notification that arrives after the event loop closed instead of raising on the reload thread. --- ldclient/async_client.py | 14 ++++++++------ ldclient/client.py | 14 ++++++++------ ldclient/impl/datasystem/async_fdv2.py | 7 ++++++- ldclient/impl/events/types.py | 2 ++ 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/ldclient/async_client.py b/ldclient/async_client.py index f5008f5f..11c1461a 100644 --- a/ldclient/async_client.py +++ b/ldclient/async_client.py @@ -558,11 +558,13 @@ def not_ready() -> Tuple[EvaluationDetail, Optional[FeatureFlag]]: log.error("Unexpected error while evaluating feature flag \"%s\": %s" % (key, repr(e))) log.debug(traceback.format_exc()) reason = error_reason('EXCEPTION') - # The evaluation read the flag definition before it failed, so a definition from - # the override layer marks the result. - if is_override_definition(flag): + # The evaluation read the flag definition before it failed. A definition from + # the override layer marks the result, so the failure is reported in the + # summary counters only, like any other override-affected evaluation. + override_affected = is_override_definition(flag) + if override_affected: reason['overrideAffected'] = True - self._send_event(event_factory.new_default_event(flag, context, default, reason)) + self._send_event(event_factory.new_default_event(flag, context, default, reason, override_affected)) return EvaluationDetail(default, None, reason), flag async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: @@ -659,11 +661,11 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState reason = error_reason('EXCEPTION') # The evaluation read the flag definition before it failed, so a definition from # the override layer marks the result. - if is_override_definition(flag): + override_affected = is_override_definition(flag) + if override_affected: reason['overrideAffected'] = True detail = EvaluationDetail(None, None, reason) prerequisites = [] - override_affected = False requires_experiment_data = EventFactory.is_experiment(flag, detail.reason) track_events = flag.get('trackEvents', False) or requires_experiment_data track_reason = requires_experiment_data diff --git a/ldclient/client.py b/ldclient/client.py index d26ba507..e670f04e 100644 --- a/ldclient/client.py +++ b/ldclient/client.py @@ -545,11 +545,13 @@ def not_ready() -> Tuple[EvaluationDetail, Optional[FeatureFlag]]: log.error("Unexpected error while evaluating feature flag \"%s\": %s" % (key, repr(e))) log.debug(traceback.format_exc()) reason = error_reason('EXCEPTION') - # The evaluation read the flag definition before it failed, so a definition from - # the override layer marks the result. - if is_override_definition(flag): + # The evaluation read the flag definition before it failed. A definition from + # the override layer marks the result, so the failure is reported in the + # summary counters only, like any other override-affected evaluation. + override_affected = is_override_definition(flag) + if override_affected: reason['overrideAffected'] = True - self._send_event(event_factory.new_default_event(flag, context, default, reason)) + self._send_event(event_factory.new_default_event(flag, context, default, reason, override_affected)) return EvaluationDetail(default, None, reason), flag def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: @@ -652,11 +654,11 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: reason = error_reason('EXCEPTION') # The evaluation read the flag definition before it failed, so a definition from # the override layer marks the result. - if is_override_definition(flag): + override_affected = is_override_definition(flag) + if override_affected: reason['overrideAffected'] = True detail = EvaluationDetail(None, None, reason) prerequisites = [] - override_affected = False requires_experiment_data = EventFactory.is_experiment(flag, detail.reason) track_events = flag.get('trackEvents', False) or requires_experiment_data diff --git a/ldclient/impl/datasystem/async_fdv2.py b/ldclient/impl/datasystem/async_fdv2.py index 7592d02c..a0db7c3d 100644 --- a/ldclient/impl/datasystem/async_fdv2.py +++ b/ldclient/impl/datasystem/async_fdv2.py @@ -347,7 +347,12 @@ def start(self, set_on_ready: AsyncEvent): listeners = self._flag_change_listeners def notify(key: str) -> None: - loop.call_soon_threadsafe(listeners.notify, FlagChange(key)) + try: + loop.call_soon_threadsafe(listeners.notify, FlagChange(key)) + except RuntimeError: + # A reload can finish after the client closed. The loop is closed then, + # and the notification has nobody left to reach. + log.debug("Dropped the flag change notification for %s because the event loop is closed", key) self._override_sink = OverrideSinkImpl(self._override_layer, self._store._memory_store, notify, listeners.has_listeners) try: diff --git a/ldclient/impl/events/types.py b/ldclient/impl/events/types.py index b107a3b7..6f754a23 100644 --- a/ldclient/impl/events/types.py +++ b/ldclient/impl/events/types.py @@ -186,6 +186,7 @@ def new_default_event( context: Context, default_value: Any, reason: Optional[dict], + override_affected: bool = False, ) -> EventInputEvaluation: return EventInputEvaluation( self._timestamp_fn(), @@ -198,6 +199,7 @@ def new_default_event( default_value, None, flag.track_events, + override_affected, ) def new_unknown_flag_event( From 9e6e59880d69613622d1ae228c1bc96d66b55006 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Sat, 3 Oct 2026 00:51:12 +0000 Subject: [PATCH 13/26] test: Run the override vectors and the event marking scenarios on the async client --- .../test_async_client_override_vectors.py | 71 +++++++ .../testing/test_async_client_overrides.py | 189 ++++++++++++++++++ ldclient/testing/test_ldclient_overrides.py | 36 +++- 3 files changed, 295 insertions(+), 1 deletion(-) create mode 100644 ldclient/testing/test_async_client_override_vectors.py diff --git a/ldclient/testing/test_async_client_override_vectors.py b/ldclient/testing/test_async_client_override_vectors.py new file mode 100644 index 00000000..3eab7e3f --- /dev/null +++ b/ldclient/testing/test_async_client_override_vectors.py @@ -0,0 +1,71 @@ +""" +Runs the OVERRIDE specification test vectors through the async client. The vectors and the +checks are those of the sync runner: each vector sets up LaunchDarkly data, an override layer, +and an initialization state, and the test evaluates one flag through the full client stack. +""" +from typing import Any, Dict + +import pytest + +from ldclient.async_client import AsyncLDClient +from ldclient.async_config import AsyncConfig, AsyncDataSystemConfig +from ldclient.context import Context +from ldclient.impl.events.types import EventInputEvaluation +from ldclient.testing.mock_async_components import MockAsyncEventProcessor +from ldclient.testing.mock_components import ( + MockDataSourceBuilder, + MockOverrideSource +) +from ldclient.testing.test_async_client_overrides import ( + AsyncHangingSynchronizer, + AsyncStaticInitializer +) +from ldclient.testing.test_ldclient_override_vectors import ( + assert_reason, + load_vectors, + override_flags, + vector_id +) + + +async def make_client(vector: Dict[str, Any]) -> AsyncLDClient: + source = MockOverrideSource(flags=override_flags(vector['overrides']), segments=dict(vector['overrides'].get('segments', {}))) + ld_data = vector['launchDarklyData'] + if ld_data['initialized']: + initializer = AsyncStaticInitializer(ld_data.get('flags', {}), ld_data.get('segments', {})) + datasystem = AsyncDataSystemConfig(initializers=[MockDataSourceBuilder(initializer)], override_source=source.builder) + start_wait = 5 + else: + # With no sources at all, the client would consider cached data available rather than + # applying its not-initialized handling. A synchronizer that never delivers avoids that. + datasystem = AsyncDataSystemConfig(synchronizers=[MockDataSourceBuilder(AsyncHangingSynchronizer())], override_source=source.builder) + start_wait = 0 + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor()) + client = AsyncLDClient(config) + await client.start(start_wait=start_wait) + return client + + +@pytest.mark.asyncio +@pytest.mark.parametrize("vector", load_vectors(), ids=vector_id) +async def test_override_spec_vector(vector: Dict[str, Any]): + client = await make_client(vector) + try: + assert await client.is_initialized() is vector['launchDarklyData']['initialized'] + evaluate = vector['evaluate'] + detail = await client.variation_detail(evaluate['flagKey'], Context.from_dict(evaluate['context']), evaluate['defaultValue']) + + expect = vector['expect'] + assert detail.value == expect['value'], "value" + assert detail.variation_index == expect['variationIndex'], "variationIndex" + assert_reason(expect['reason'], detail.reason) + + # summaryOverrideAffected is the marking the client hands to the event processor for this + # evaluation. The event processor keys individual-event suppression and the summary + # counter marker on that scalar, not on the reason. + if 'summaryOverrideAffected' in expect: + records = [e for e in client._event_processor.events if isinstance(e, EventInputEvaluation) and e.key == evaluate['flagKey']] + assert len(records) == 1, "expected exactly one evaluation record for the flag" + assert records[0].override_affected is expect['summaryOverrideAffected'], "summaryOverrideAffected" + finally: + await client.close() diff --git a/ldclient/testing/test_async_client_overrides.py b/ldclient/testing/test_async_client_overrides.py index 3556d9bf..18a33554 100644 --- a/ldclient/testing/test_async_client_overrides.py +++ b/ldclient/testing/test_async_client_overrides.py @@ -3,6 +3,7 @@ client tests. """ import asyncio +import json import logging import threading import time @@ -15,11 +16,17 @@ from ldclient.async_feature_store import AsyncInMemoryFeatureStore from ldclient.context import Context from ldclient.evaluation import EvaluationDetail +from ldclient.hook import AsyncHook, EvaluationSeriesContext, Metadata from ldclient.impl.aio.concurrency import AsyncEvent +from ldclient.impl.events.async_event_processor import ( + DefaultAsyncEventProcessor +) from ldclient.impl.events.types import EventInputEvaluation from ldclient.impl.integrations.files.filedata import make_flag_with_value from ldclient.interfaces import AsyncFeatureStore, DataStoreMode +from ldclient.migrations import Stage from ldclient.testing.builders import FlagBuilder +from ldclient.testing.impl.events.test_async_event_processor import MockAioHttp from ldclient.testing.mock_async_components import MockAsyncEventProcessor from ldclient.testing.mock_components import ( FailingOverrideSourceBuilder, @@ -472,3 +479,185 @@ async def test_all_flags_state_turns_off_event_tracking_for_override_affected_fl assert state.get_flag_value('overridden-flag') is True finally: await client.close() + + +@pytest.mark.asyncio +async def test_override_source_is_not_started_when_offline(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + config = AsyncConfig('SDK_KEY', datasystem_config=AsyncDataSystemConfig(override_source=source.builder), offline=True) + client = AsyncLDClient(config) + await client.start(start_wait=0) + try: + assert source.start_count == 0 + assert await client.variation('overridden-flag', user, False) is False + finally: + await client.close() + + +def test_override_update_after_the_client_and_its_loop_are_gone_is_dropped(): + source = MockOverrideSource() + + async def run_client() -> None: + client = await make_uninitialized_client(source) + client.flag_tracker.add_listener(lambda change: None) + await client.close() + + asyncio.run(run_client()) + # A reload that finishes after the client closed finds the event loop closed. The + # notification has nobody left to reach and is dropped. + source.set_overrides({'overridden-flag': single_value_flag('overridden-flag', True)}, {}) + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +def tracked_bool_flag(key: str) -> FlagBuilder: + return FlagBuilder(key).version(100).variations(False, True).off_variation(0).fallthrough_variation(1).track_events(True) + + +def evaluation_events_by_key(client: AsyncLDClient) -> Dict[str, EventInputEvaluation]: + """The evaluation records the client handed to the event processor, keyed by flag key.""" + processor: Any = client._event_processor + return {event.key: event for event in processor.events if isinstance(event, EventInputEvaluation)} + + +@pytest.mark.asyncio +async def test_overridden_prerequisite_marks_the_dependent_evaluation_records(): + # top-flag (LaunchDarkly) --> mid-flag (LaunchDarkly) --> leaf-flag (overridden) + # --> plain-flag (LaunchDarkly) + # The LaunchDarkly copy of leaf-flag is off, so the chain passes only through the override. + ld_data = { + 'top-flag': tracked_bool_flag('top-flag').on(True).prerequisite('mid-flag', 1).prerequisite('plain-flag', 1).build().to_json_dict(), + 'mid-flag': tracked_bool_flag('mid-flag').on(True).prerequisite('leaf-flag', 1).build().to_json_dict(), + 'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict(), + 'leaf-flag': tracked_bool_flag('leaf-flag').on(False).build().to_json_dict(), + } + source = MockOverrideSource(flags={'leaf-flag': tracked_bool_flag('leaf-flag').on(True).build().to_json_dict()}) + client = await make_initialized_client(ld_data, source) + try: + detail = await client.variation_detail('top-flag', user, False) + assert detail.value is True + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + + records = evaluation_events_by_key(client) + assert sorted(records.keys()) == ['leaf-flag', 'mid-flag', 'plain-flag', 'top-flag'] + assert records['top-flag'].override_affected is True + assert records['mid-flag'].override_affected is True + assert records['leaf-flag'].override_affected is True + assert records['plain-flag'].override_affected is False + assert records['leaf-flag'].reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + assert records['plain-flag'].reason == {'kind': 'FALLTHROUGH'} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_wrong_type_result_of_overridden_flag_stays_marked(): + details = [] + + class CapturingHook(AsyncHook): + @property + def metadata(self) -> Metadata: + return Metadata(name='capturing-hook') + + async def before_evaluation(self, series_context: EvaluationSeriesContext, data: dict) -> dict: + return data + + async def after_evaluation(self, series_context: EvaluationSeriesContext, data: dict, detail: EvaluationDetail) -> dict: + details.append(detail) + return data + + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', 'not-a-stage')}) + datasystem = AsyncDataSystemConfig(synchronizers=[MockDataSourceBuilder(AsyncHangingSynchronizer())], override_source=source.builder) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor(), hooks=[CapturingHook()]) + client = AsyncLDClient(config) + await client.start(start_wait=0) + try: + stage, _ = await client.migration_variation('overridden-flag', user, Stage.OFF) + assert stage == Stage.OFF + assert len(details) == 1 + assert details[0].value == 'off' + assert details[0].reason == {'kind': 'ERROR', 'errorKind': 'WRONG_TYPE', 'overrideAffected': True} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_failed_evaluation_is_marked_only_when_the_flag_came_from_the_override_layer(monkeypatch): + source = MockOverrideSource(flags={'overridden-flag': tracked_bool_flag('overridden-flag').on(True).build().to_json_dict()}) + client = await make_initialized_client({'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict()}, source) + try: + monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure) + + # The failure of the override flag is marked. The failure of the ordinary flag is not. + detail = await client.variation_detail('overridden-flag', user, 'default') + assert detail == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True}) + detail = await client.variation_detail('plain-flag', user, 'default') + assert detail == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION'}) + + # The marked record produces no individual event. The ordinary record keeps its tracking. + records = evaluation_events_by_key(client) + assert records['overridden-flag'].override_affected is True + assert records['plain-flag'].override_affected is False + assert records['plain-flag'].track_events is True + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_all_flags_state_turns_off_event_tracking_for_an_override_flag_whose_evaluation_fails(monkeypatch): + source = MockOverrideSource(flags={'overridden-flag': tracked_bool_flag('overridden-flag').on(True).build().to_json_dict()}) + client = await make_initialized_client({'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict()}, source) + try: + monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure) + state = await client.all_flags_state(user, with_reasons=True) + + # The failed override flag stays in the state with a marked reason and no tracking + # fields. The failed ordinary flag keeps its tracking fields. + assert state.valid is True + flags_state = state.to_json_dict()['$flagsState'] + assert flags_state['overridden-flag']['reason'] == {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True} + assert 'trackEvents' not in flags_state['overridden-flag'] + assert flags_state['plain-flag']['reason'] == {'kind': 'ERROR', 'errorKind': 'EXCEPTION'} + assert flags_state['plain-flag']['trackEvents'] is True + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_override_affected_evaluations_appear_only_in_summary_output(): + # End to end through the real event processor: the overridden flag requests individual + # feature events and debug events, and an ordinary flag requests feature events. + debug_until = int(time.time() * 1000) + 100000 + overridden = FlagBuilder('flag-tracked-override').version(300).on(False).off_variation(0).variations('override-value').track_events(True).debug_events_until_date(debug_until).build().to_json_dict() + normal = FlagBuilder('flag-normal').version(100).on(False).off_variation(0).variations('normal-value').track_events(True).build().to_json_dict() + source = MockOverrideSource(flags={'flag-tracked-override': overridden}) + mock_http = MockAioHttp() + + initializer = AsyncStaticInitializer({'flag-normal': normal}, {}) + datasystem = AsyncDataSystemConfig(initializers=[MockDataSourceBuilder(initializer)], override_source=source.builder) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, diagnostic_opt_out=True, event_processor_class=lambda config: DefaultAsyncEventProcessor(config, mock_http)) + client = AsyncLDClient(config) + await client.start(start_wait=5) + try: + assert await client.is_initialized() is True + for _ in range(2): + assert await client.variation('flag-tracked-override', user, 'default1') == 'override-value' + assert await client.variation('flag-normal', user, 'default2') == 'normal-value' + assert await client._event_processor.flush_and_wait(5) is True + finally: + await client.close() + + assert mock_http.request_data is not None + output = json.loads(mock_http.request_data) + kinds = sorted(e['kind'] for e in output) + assert kinds == ['feature', 'index', 'summary'] + feature = [e for e in output if e['kind'] == 'feature'][0] + assert feature['key'] == 'flag-normal' + summary = [e for e in output if e['kind'] == 'summary'][0] + assert summary['features']['flag-tracked-override']['default'] == 'default1' + assert summary['features']['flag-tracked-override']['counters'] == [ + {'count': 2, 'value': 'override-value', 'variation': 0, 'version': 300, 'overrideAffected': True} + ] + assert summary['features']['flag-normal']['counters'] == [{'count': 1, 'value': 'normal-value', 'variation': 0, 'version': 100}] diff --git a/ldclient/testing/test_ldclient_overrides.py b/ldclient/testing/test_ldclient_overrides.py index 3f68134e..78d0a1f9 100644 --- a/ldclient/testing/test_ldclient_overrides.py +++ b/ldclient/testing/test_ldclient_overrides.py @@ -585,7 +585,7 @@ def test_all_flags_state_turns_off_event_tracking_for_override_affected_flags(): assert flags_state['overridden-flag']['version'] == 7 -def test_all_flags_state_keeps_details_of_override_affected_flags_when_details_only_for_tracked_flags(): +def test_all_flags_state_omits_details_of_override_affected_flags_when_details_only_for_tracked_flags(): # With details only for tracked flags, an override-affected flag counts as untracked, so its # version and reason are omitted like any other untracked flag, and its value stays. overridden = FlagBuilder('overridden-flag').version(7).on(False).off_variation(0).variations(True).track_events(True).build().to_json_dict() @@ -624,6 +624,40 @@ def after_evaluation(self, series_context: EvaluationSeriesContext, data: dict, assert details[0].reason == {'kind': 'ERROR', 'errorKind': 'WRONG_TYPE', 'overrideAffected': True} +def test_failed_evaluation_is_marked_only_when_the_flag_came_from_the_override_layer(monkeypatch): + source = MockOverrideSource(flags={'overridden-flag': tracked_bool_flag('overridden-flag').on(True).build().to_json_dict()}) + with make_initialized_client({'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict()}, source) as client: + monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure) + + # The failure of the override flag is marked. The failure of the ordinary flag is not. + detail = client.variation_detail('overridden-flag', user, 'default') + assert detail == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True}) + detail = client.variation_detail('plain-flag', user, 'default') + assert detail == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION'}) + + # The marked record produces no individual event. The ordinary record keeps its tracking. + records = evaluation_events_by_key(client) + assert records['overridden-flag'].override_affected is True + assert records['plain-flag'].override_affected is False + assert records['plain-flag'].track_events is True + + +def test_all_flags_state_turns_off_event_tracking_for_an_override_flag_whose_evaluation_fails(monkeypatch): + source = MockOverrideSource(flags={'overridden-flag': tracked_bool_flag('overridden-flag').on(True).build().to_json_dict()}) + with make_initialized_client({'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict()}, source) as client: + monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure) + state = client.all_flags_state(user, with_reasons=True) + + # The failed override flag stays in the state with a marked reason and no tracking + # fields. The failed ordinary flag keeps its tracking fields. + assert state.valid is True + flags_state = state.to_json_dict()['$flagsState'] + assert flags_state['overridden-flag']['reason'] == {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True} + assert 'trackEvents' not in flags_state['overridden-flag'] + assert flags_state['plain-flag']['reason'] == {'kind': 'ERROR', 'errorKind': 'EXCEPTION'} + assert flags_state['plain-flag']['trackEvents'] is True + + def test_override_affected_evaluations_appear_only_in_summary_output(): # End to end through the real event processor: the overridden flag requests individual # feature events and debug events, and an ordinary flag requests feature events. From cad8f0b7f10e8f6bed7686e0432f6f0b787d68fc Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:11:37 +0000 Subject: [PATCH 14/26] test: Drop the async offline override test that the override store change now covers --- ldclient/testing/test_async_client_overrides.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/ldclient/testing/test_async_client_overrides.py b/ldclient/testing/test_async_client_overrides.py index 18a33554..8fe69458 100644 --- a/ldclient/testing/test_async_client_overrides.py +++ b/ldclient/testing/test_async_client_overrides.py @@ -481,19 +481,6 @@ async def test_all_flags_state_turns_off_event_tracking_for_override_affected_fl await client.close() -@pytest.mark.asyncio -async def test_override_source_is_not_started_when_offline(): - source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) - config = AsyncConfig('SDK_KEY', datasystem_config=AsyncDataSystemConfig(override_source=source.builder), offline=True) - client = AsyncLDClient(config) - await client.start(start_wait=0) - try: - assert source.start_count == 0 - assert await client.variation('overridden-flag', user, False) is False - finally: - await client.close() - - def test_override_update_after_the_client_and_its_loop_are_gone_is_dropped(): source = MockOverrideSource() From f30883f210bd07646ad081ee06d6323aab4e336d Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:29:38 -0700 Subject: [PATCH 15/26] feat: Add the file-based override source Adds ldclient.integrations.overrides.FileOverrideSourceBuilder, the file-based override source described by the OVERRIDE specification. The source reads one or more JSON or YAML files in the file data source document format, with optional flags, flagValues, and segments members, and supplies each successful load to the SDK's override sink as a full snapshot. Files are combined in the configured order. The duplicate keys handling is fail by default, which rejects the reload and keeps the previously loaded overrides, or ignore, which keeps the first file's entry. A configured file that does not exist contributes no overrides, so a file can be created later and deleting a file removes its overrides. A file that exists but cannot be read or parsed fails that reload, the last good overrides stay in effect, the failure is logged, and the load is retried after a bounded delay and on the next detected change. Change detection is one of two modes: polling, the default, examines the files once per second by default with a one second minimum, and watching reacts to file system notifications through the watchdog package. Watching without the watchdog package and a builder with no paths are construction errors. The initial load completes during client construction. Every applied change is logged at Info level with the overrides in effect and what each configured file supplied. --- docs/api-integrations.rst | 10 + .../impl/integrations/overrides/__init__.py | 0 .../overrides/file_override_source.py | 148 ++++++ ldclient/integrations/overrides.py | 152 ++++++ .../integrations/test_file_override_source.py | 441 ++++++++++++++++++ 5 files changed, 751 insertions(+) create mode 100644 ldclient/impl/integrations/overrides/__init__.py create mode 100644 ldclient/impl/integrations/overrides/file_override_source.py create mode 100644 ldclient/integrations/overrides.py create mode 100644 ldclient/testing/integrations/test_file_override_source.py diff --git a/docs/api-integrations.rst b/docs/api-integrations.rst index 8d8146ff..594d7ed8 100644 --- a/docs/api-integrations.rst +++ b/docs/api-integrations.rst @@ -8,3 +8,13 @@ ldclient.integrations module :members: :special-members: __init__ :show-inheritance: + +ldclient.integrations.overrides module +-------------------------------------- + +The entry point for this feature is :class:`ldclient.integrations.overrides.FileOverrideSourceBuilder`. + +.. automodule:: ldclient.integrations.overrides + :members: + :special-members: __init__ + :show-inheritance: diff --git a/ldclient/impl/integrations/overrides/__init__.py b/ldclient/impl/integrations/overrides/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ldclient/impl/integrations/overrides/file_override_source.py b/ldclient/impl/integrations/overrides/file_override_source.py new file mode 100644 index 00000000..f9367701 --- /dev/null +++ b/ldclient/impl/integrations/overrides/file_override_source.py @@ -0,0 +1,148 @@ +""" +The file-based override source. It reads flag and segment overrides from one or more local +files and reloads them when the files change. +""" + +import threading +from enum import Enum +from typing import List, Optional, Union + +from ldclient.impl.integrations.files.filedata import ( + DEFAULT_DEBOUNCE_DELAY, + DEFAULT_RETRY_DELAY, + DuplicateKeysHandling, + FileSummary, + MergeResult, + Poller, + Reloader, + Watcher +) +from ldclient.impl.util import log +from ldclient.interfaces import OverrideSink, OverrideSource + + +class ChangeDetection(str, Enum): + """ + Selects how the file-based override source learns that a file changed. The two modes are + alternatives. Flag overrides are currently experimental and subject to change. + """ + + POLLING = "polling" + """ + The source examines the files on a fixed interval and reloads when the modification time + or the size of a file changes. Polling works on every file system, including network + mounts and directories whose contents are swapped through symbolic links, as Kubernetes + does for mounted ConfigMaps. It is the default. + """ + + WATCHING = "watching" + """ + The source reloads in response to file system change notifications, using the ``watchdog`` + package. It reacts faster than polling. It depends on notifications, which some file + systems do not deliver reliably. + """ + + +class _FileOverrideSource(OverrideSource): + """ + Reads overrides from the configured files and supplies each successful load to the sink as + a full snapshot. A configured file that does not exist contributes no overrides. A file + that cannot be read or parsed fails that load, the last good snapshot stays in effect, and + the load is retried. + """ + + def __init__(self, paths: List[str], duplicate_keys_handling: DuplicateKeysHandling, change_detection: ChangeDetection, poll_interval: float): + self._paths = paths + self._duplicate_keys_handling = duplicate_keys_handling + self._change_detection = change_detection + self._poll_interval = poll_interval + self._reloader: Optional[Reloader] = None + self._change_detector: Optional[Union[Poller, Watcher]] = None + self._lock = threading.Lock() + self._closed = False + + def start(self, sink: OverrideSink) -> None: + def apply(merged: MergeResult) -> None: + sink.set_overrides(merged.flags, merged.segments) + _log_overrides_in_effect(merged) + + reloader = Reloader( + self._paths, + self._duplicate_keys_handling, + apply=apply, + skip_missing_paths=True, + debounce_delay=DEFAULT_DEBOUNCE_DELAY, + retry_delay=DEFAULT_RETRY_DELAY, + skip_unchanged=True, + ) + with self._lock: + if self._closed: + return + self._reloader = reloader + + # The initial load runs synchronously, so overrides present in the files are in effect + # by the time the client constructor returns. A file that does not exist yet + # contributes no overrides. A file that cannot be read or parsed is not fatal: the + # client runs with no overrides, the failure is logged, and the retry recovers once + # the file is readable. + reloader.reload_now() + + with self._lock: + if self._closed: + return + if self._change_detection == ChangeDetection.WATCHING: + self._change_detector = Watcher(self._paths, reloader.trigger) + else: + poller = Poller(self._paths, self._poll_interval, reloader.trigger) + poller.start() + self._change_detector = poller + + def close(self) -> None: + with self._lock: + if self._closed: + return + self._closed = True + change_detector = self._change_detector + reloader = self._reloader + self._change_detector = None + self._reloader = None + if change_detector is not None: + change_detector.close() + if reloader is not None: + reloader.close() + + +def _log_overrides_in_effect(merged: MergeResult) -> None: + """ + Reports the overrides now in effect and what each configured file supplied. The reloader + applies a snapshot only when the content changed, so this logs each change once. + """ + details = "; ".join(_file_summary_text(summary) for summary in merged.files) + if len(merged.flags) == 0 and len(merged.segments) == 0: + log.info("Flag overrides: none in effect (%s)", details) + return + log.info("Flag overrides in effect: %s (%s)", _counts_text(len(merged.flags), len(merged.segments)), details) + + +def _file_summary_text(summary: FileSummary) -> str: + if not summary.present: + return "%s: absent" % summary.path + if summary.flags == 0 and summary.segments == 0: + return "%s: no entries" % summary.path + return "%s: %s" % (summary.path, _counts_text(summary.flags, summary.segments)) + + +def _counts_text(flags: int, segments: int) -> str: + """Formats flag and segment counts, for example "2 flags, 1 segment".""" + parts = [] + if flags > 0: + parts.append(_pluralize(flags, "flag")) + if segments > 0: + parts.append(_pluralize(segments, "segment")) + return ", ".join(parts) + + +def _pluralize(count: int, noun: str) -> str: + if count == 1: + return "1 %s" % noun + return "%d %ss" % (count, noun) diff --git a/ldclient/integrations/overrides.py b/ldclient/integrations/overrides.py new file mode 100644 index 00000000..6933253b --- /dev/null +++ b/ldclient/integrations/overrides.py @@ -0,0 +1,152 @@ +""" +Override sources for the SDK's flag override capability. Flag overrides are currently +experimental and subject to change. + +Overrides are flag and segment definitions that take precedence over data received from +LaunchDarkly at evaluation time, on a per-key basis. They exist for resilience during an +incident. An operator can force one or more flags to a known state on a running application, +whether or not the application can reach LaunchDarkly. The override stays in effect until the +operator removes it. Flags not present in the override data are completely unaffected. + +This module currently provides one source: :class:`FileOverrideSourceBuilder`, which reads +overrides from local files and reloads them as the files change. Configure it with the data +system builder: +:: + + from ldclient import Config, datasystem + from ldclient.integrations.overrides import FileOverrideSourceBuilder + + source = FileOverrideSourceBuilder(['/etc/launchdarkly/overrides.json']) + config = Config(sdk_key, datasystem_config=datasystem.default().overrides(source).build()) + +An evaluation that an override affects is marked. The marking is direct or transitive. It +applies when the evaluated flag, a prerequisite at any depth, or a segment read during the +evaluation came from the override layer. The evaluation reason's ``overrideAffected`` property +reports the marking. Marked evaluations appear in analytics summary events only, under separate +counters, so LaunchDarkly can distinguish them. They produce no individual evaluation events. +""" + +from typing import List, Optional, Union + +from ldclient.config import DataSourceBuilderConfig, OverrideSourceBuilder +from ldclient.impl.integrations.files.filedata import ( + DuplicateKeysHandling, + abs_file_paths, + have_watchdog +) +from ldclient.impl.integrations.overrides.file_override_source import ( + ChangeDetection, + _FileOverrideSource +) +from ldclient.impl.util import log +from ldclient.interfaces import OverrideSource + + +class FileOverrideSourceBuilder(OverrideSourceBuilder): + """ + A builder for a file-based override source. Flag overrides are currently experimental and + subject to change. + + The source reads flag and segment overrides from one or more local files and reloads them + as the files change. The files use the same document format as the file data source: + each file is a JSON or YAML document with optional ``flags``, ``flagValues``, and + ``segments`` members. ``flagValues`` entries are expanded into full flag definitions that + return the given value for every context. YAML requires the ``pyyaml`` package. + + When multiple files are configured, their entries are combined in the configured order. + The order determines which file wins under the duplicate keys handling. A reload replaces + the entire override set, so removing an entry from the files removes the override. A + configured file that does not exist contributes no overrides: deleting a file removes its + overrides, and deleting every file removes them all. A file that exists but cannot be read + or parsed makes that whole reload fail. The previously loaded overrides stay in effect, the + source logs the failure, retries after a short delay, and recovers on its own once the file + is readable again. + + Whenever the set of overrides in effect changes, including at startup, the source logs the + overrides in effect and what each configured file supplied, at Info level. + + By default the source polls the files for changes once per second. See + :meth:`change_detection` and :meth:`poll_interval`. + """ + + DEFAULT_POLL_INTERVAL = 1.0 + """ + The interval, in seconds, at which the source examines the files for changes in polling + mode when no interval was specified. Because the source reads local files rather than + contacting a service, a short interval keeps an override responsive during an incident at + negligible cost. + """ + + MINIMUM_POLL_INTERVAL = 1.0 + """ + The shortest allowed polling interval, in seconds. A configured interval below this is + raised to it. The minimum exists only to prevent a pathological tight loop over the file + system. + """ + + def __init__(self, paths: Union[str, List[str]]): + """ + :param paths: the files to load overrides from, as a single path or a list of paths. + The order is significant: it determines which file wins under the duplicate keys + handling when the same key appears in more than one file. Relative paths are + resolved against the current working directory when the source is built. + """ + self.__paths: List[str] = [paths] if isinstance(paths, str) else list(paths) + self.__duplicate_keys_handling = DuplicateKeysHandling.FAIL + self.__change_detection = ChangeDetection.POLLING + self.__poll_interval = self.DEFAULT_POLL_INTERVAL + + def duplicate_keys_handling(self, handling: Union[DuplicateKeysHandling, str]) -> 'FileOverrideSourceBuilder': + """ + Specifies how to handle the same key appearing in more than one file. The default is + :attr:`DuplicateKeysHandling.FAIL`, which treats the reload as failed and keeps the + previously loaded overrides. :attr:`DuplicateKeysHandling.IGNORE` keeps the entry from + the first configured file that defines the key and discards the others. + + :param handling: the handling, as the enum or its string value (``"fail"`` or ``"ignore"``) + """ + self.__duplicate_keys_handling = DuplicateKeysHandling(handling) + return self + + def change_detection(self, mode: Union[ChangeDetection, str]) -> 'FileOverrideSourceBuilder': + """ + Selects how the source detects file changes. The default is + :attr:`ChangeDetection.POLLING`. The two modes are alternatives, so setting one replaces + the other. :attr:`ChangeDetection.WATCHING` requires the ``watchdog`` package. + + :param mode: the mode, as the enum or its string value (``"polling"`` or ``"watching"``) + """ + self.__change_detection = ChangeDetection(mode) + return self + + def poll_interval(self, seconds: float) -> 'FileOverrideSourceBuilder': + """ + Sets the interval between examinations of the files in polling mode. Watching mode + ignores it. The default is :attr:`DEFAULT_POLL_INTERVAL`. An interval below + :attr:`MINIMUM_POLL_INTERVAL` is raised to the minimum. + + :param seconds: the interval in seconds + """ + self.__poll_interval = seconds + return self + + def build(self, config: DataSourceBuilderConfig) -> OverrideSource: # pylint: disable=unused-argument + """ + Builds the override source. This is called internally by the SDK. It raises + ``ValueError`` when no file paths were specified or when watching mode was selected + without the ``watchdog`` package. + """ + if len(self.__paths) == 0: + raise ValueError("no file paths were specified for the file-based override source") + if self.__change_detection == ChangeDetection.WATCHING and not have_watchdog: + raise ValueError("the file-based override source cannot watch files for changes because the watchdog package is not installed; install it or use polling") + + poll_interval = self.__poll_interval + if self.__change_detection == ChangeDetection.POLLING and poll_interval < self.MINIMUM_POLL_INTERVAL: + log.warning("Poll interval %s is below the minimum for the file-based override source; using %s", poll_interval, self.MINIMUM_POLL_INTERVAL) + poll_interval = self.MINIMUM_POLL_INTERVAL + + return _FileOverrideSource(abs_file_paths(self.__paths), self.__duplicate_keys_handling, self.__change_detection, poll_interval) + + +__all__ = ['ChangeDetection', 'DuplicateKeysHandling', 'FileOverrideSourceBuilder'] diff --git a/ldclient/testing/integrations/test_file_override_source.py b/ldclient/testing/integrations/test_file_override_source.py new file mode 100644 index 00000000..ab940a85 --- /dev/null +++ b/ldclient/testing/integrations/test_file_override_source.py @@ -0,0 +1,441 @@ +""" +Tests for the file-based override source: its builder, initial load, multi-file merging, +absent files, change detection in both modes, failure retention, and its Info log. +""" +import logging +import os +import time +from queue import Empty, Queue +from typing import Any, Dict, List, Optional, Tuple + +import pytest + +from ldclient.client import Config, Context, LDClient +from ldclient.config import Config as SDKConfig +from ldclient.datasystem import custom +from ldclient.impl.integrations.files import filedata +from ldclient.impl.integrations.overrides.file_override_source import ( + _FileOverrideSource +) +from ldclient.integrations.overrides import ( + ChangeDetection, + DuplicateKeysHandling, + FileOverrideSourceBuilder +) +from ldclient.testing.mock_components import HangingSynchronizer +from ldclient.testing.stub_util import MockEventProcessor + +TEST_TIMEOUT = 10.0 +QUIET_PERIOD = 0.3 + +watchdog_required = pytest.mark.skipif(not filedata.have_watchdog, reason="watchdog is not installed") +yaml_required = pytest.mark.skipif(not filedata.have_yaml, reason="pyyaml is not installed") + +Snapshot = Tuple[Dict[str, Any], Dict[str, Any]] + + +class CapturingSink: + """Records every override snapshot it receives.""" + + def __init__(self): + self.snapshots: Queue = Queue() + + def set_overrides(self, flags, segments) -> None: + self.snapshots.put((dict(flags), dict(segments))) + + def require_snapshot(self, timeout: float = TEST_TIMEOUT) -> Snapshot: + try: + return self.snapshots.get(timeout=timeout) + except Empty: + pytest.fail("timed out waiting for an override snapshot") + + def require_no_snapshot(self, duration: float = QUIET_PERIOD) -> None: + try: + snapshot = self.snapshots.get(timeout=duration) + except Empty: + return + pytest.fail("received an unexpected override snapshot: %r" % (snapshot,)) + + +def write_file(path: str, content: str) -> None: + with open(path, 'w') as f: + f.write(content) + + +def flag_values(snapshot: Snapshot) -> Dict[str, Any]: + """The single variation of each flag in a snapshot, keyed by flag key.""" + return {key: flag.variations[0] for key, flag in snapshot[0].items()} + + +@pytest.fixture +def sources(): + started: List[_FileOverrideSource] = [] + yield started + for source in started: + source.close() + + +def build_source(sources, paths, configure=None) -> Tuple[_FileOverrideSource, CapturingSink]: + builder = FileOverrideSourceBuilder(paths) + if configure is not None: + configure(builder) + source = builder.build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + sink = CapturingSink() + source.start(sink) + sources.append(source) + return source, sink + + +def info_lines(caplog) -> List[str]: + return [r.getMessage() for r in caplog.records if r.levelno == logging.INFO] + + +def require_info_line(caplog, expected: str) -> None: + """Waits for an Info line equal to the expected text. The source logs after it hands the snapshot to the sink.""" + deadline = time.time() + TEST_TIMEOUT + while True: + if expected in info_lines(caplog): + return + if time.time() > deadline: + pytest.fail("timed out waiting for the Info line %r; Info output: %r" % (expected, info_lines(caplog))) + time.sleep(0.01) + + +# --------------------------------------------------------------------------- +# Builder +# --------------------------------------------------------------------------- + +def test_builder_requires_paths(): + with pytest.raises(ValueError): + FileOverrideSourceBuilder([]).build(SDKConfig('SDK_KEY')) + + +def test_builder_accepts_a_single_path_string(tmp_path): + path = os.path.join(str(tmp_path), 'overrides.json') + source = FileOverrideSourceBuilder(path).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._paths == [path] + + +def test_builder_resolves_relative_paths(): + source = FileOverrideSourceBuilder(['relative/overrides.json']).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._paths == [os.path.abspath('relative/overrides.json')] + + +def test_builder_polls_by_default(tmp_path): + path = os.path.join(str(tmp_path), 'overrides.json') + source = FileOverrideSourceBuilder([path]).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._change_detection == ChangeDetection.POLLING + assert source._poll_interval == FileOverrideSourceBuilder.DEFAULT_POLL_INTERVAL + assert source._duplicate_keys_handling == DuplicateKeysHandling.FAIL + + +def test_builder_rejects_unknown_change_detection(): + with pytest.raises(ValueError) as excinfo: + FileOverrideSourceBuilder(['a']).change_detection('notify') + assert 'notify' in str(excinfo.value) + + +def test_builder_rejects_unknown_duplicate_keys_handling(): + with pytest.raises(ValueError): + FileOverrideSourceBuilder(['a']).duplicate_keys_handling('merge') + + +def test_builder_accepts_string_values(): + builder = FileOverrideSourceBuilder(['a']).change_detection('polling').duplicate_keys_handling('ignore') + source = builder.build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._change_detection == ChangeDetection.POLLING + assert source._duplicate_keys_handling == DuplicateKeysHandling.IGNORE + + +def test_builder_clamps_poll_interval_to_the_minimum(caplog): + with caplog.at_level(logging.WARNING): + source = FileOverrideSourceBuilder(['a']).poll_interval(0.001).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._poll_interval == FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL + assert any('below the minimum' in r.getMessage() for r in caplog.records) + + source = FileOverrideSourceBuilder(['a']).poll_interval(2.5).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._poll_interval == 2.5 + + +def test_builder_rejects_watching_without_watchdog(monkeypatch): + import ldclient.integrations.overrides as module + monkeypatch.setattr(module, 'have_watchdog', False) + with pytest.raises(ValueError) as excinfo: + FileOverrideSourceBuilder(['a']).change_detection(ChangeDetection.WATCHING).build(SDKConfig('SDK_KEY')) + assert 'watchdog' in str(excinfo.value) + + +# --------------------------------------------------------------------------- +# Loading +# --------------------------------------------------------------------------- + +def test_source_loads_initial_data_synchronously(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true, "flag2": "x"}, "segments": {"seg": {"key": "seg"}}}') + source, sink = build_source(sources, [path]) + # The snapshot was delivered before start returned. + snapshot = sink.snapshots.get_nowait() + assert flag_values(snapshot) == {'flag1': True, 'flag2': 'x'} + assert list(snapshot[1].keys()) == ['seg'] + assert snapshot[0]['flag1'].is_override is False, "the source supplies unmarked definitions; the SDK marks them" + + +@yaml_required +def test_source_loads_yaml(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.yaml') + write_file(path, 'flagValues:\n yaml-flag: "override-value"\n') + _, sink = build_source(sources, [path]) + assert flag_values(sink.require_snapshot()) == {'yaml-flag': 'override-value'} + + +def test_source_merges_files_in_configured_order(tmp_path, sources): + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues": {"shared": "first", "only-first": 1}}') + write_file(second, '{"flagValues": {"shared": "second", "only-second": 2}}') + _, sink = build_source(sources, [first, second], lambda b: b.duplicate_keys_handling(DuplicateKeysHandling.IGNORE)) + assert flag_values(sink.require_snapshot()) == {'shared': 'first', 'only-first': 1, 'only-second': 2} + + +def test_source_duplicate_keys_fail_by_default(tmp_path, sources, caplog): + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues": {"shared": "first"}}') + write_file(second, '{"flagValues": {"shared": "second"}}') + with caplog.at_level(logging.ERROR): + _, sink = build_source(sources, [first, second]) + # The load failed, so no snapshot was supplied and the failure was logged. + sink.require_no_snapshot() + assert any("is specified by multiple files" in r.getMessage() for r in caplog.records) + + +def test_source_starts_with_missing_file(tmp_path, sources): + path = os.path.join(str(tmp_path), 'not-yet.json') + _, sink = build_source(sources, [path], lambda b: b.poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + # A missing file contributes no overrides. The initial snapshot is empty. + assert sink.require_snapshot() == ({}, {}) + # Once the file appears, the change detection picks it up. + write_file(path, '{"flagValues": {"flag1": true}}') + assert flag_values(sink.require_snapshot()) == {'flag1': True} + + +def test_source_missing_file_contributes_no_entries(tmp_path, sources): + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues": {"from-first": true}}') + + # Step 1: one configured file exists and one does not. The existing file applies. + _, sink = build_source(sources, [first, second], lambda b: b.poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + assert flag_values(sink.require_snapshot()) == {'from-first': True} + + # Step 2: the second file appears. Both apply. + write_file(second, '{"flagValues": {"from-second": true}}') + assert flag_values(sink.require_snapshot()) == {'from-first': True, 'from-second': True} + + # Step 3: the second file is deleted. Its overrides are removed. + os.remove(second) + assert flag_values(sink.require_snapshot()) == {'from-first': True} + + # Step 4: the last file is deleted. The layer is cleared. + os.remove(first) + assert sink.require_snapshot() == ({}, {}) + + +def test_source_logs_overrides_in_effect_on_each_change(tmp_path, sources, caplog): + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues": {"flag1": true, "flag2": false}, "segments": {"seg": {"key": "seg"}}}') + + with caplog.at_level(logging.INFO): + # Step 1: at startup, one file supplies entries and the other is absent. + _, sink = build_source(sources, [first, second], lambda b: b.poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + sink.require_snapshot() + require_info_line(caplog, "Flag overrides in effect: 2 flags, 1 segment (%s: 2 flags, 1 segment; %s: absent)" % (first, second)) + + # Step 2: the absent file appears with one entry. + write_file(second, '{"flagValues": {"flag3": true}}') + sink.require_snapshot() + require_info_line(caplog, "Flag overrides in effect: 3 flags, 1 segment (%s: 2 flags, 1 segment; %s: 1 flag)" % (first, second)) + + # Step 3: both files are deleted. Nothing is in effect. + os.remove(first) + os.remove(second) + sink.require_snapshot() + require_info_line(caplog, "Flag overrides: none in effect (%s: absent; %s: absent)" % (first, second)) + + +def test_source_logs_none_in_effect_at_startup_without_files(tmp_path, sources, caplog): + path = os.path.join(str(tmp_path), 'overrides.json') + with caplog.at_level(logging.INFO): + _, sink = build_source(sources, [path]) + sink.require_snapshot() + require_info_line(caplog, "Flag overrides: none in effect (%s: absent)" % path) + + +def test_source_logs_a_present_file_with_no_entries(tmp_path, sources, caplog): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{}') + with caplog.at_level(logging.INFO): + _, sink = build_source(sources, [path]) + sink.require_snapshot() + require_info_line(caplog, "Flag overrides: none in effect (%s: no entries)" % path) + + +# --------------------------------------------------------------------------- +# Change detection +# --------------------------------------------------------------------------- + +@watchdog_required +def test_watching_mode_is_quiet_when_file_is_absent(tmp_path, sources): + path = os.path.join(str(tmp_path), 'not-yet.json') + _, sink = build_source(sources, [path], lambda b: b.change_detection(ChangeDetection.WATCHING)) + assert sink.require_snapshot() == ({}, {}) + sink.require_no_snapshot() + write_file(path, '{"flagValues": {"flag1": true}}') + assert flag_values(sink.require_snapshot()) == {'flag1': True} + + +@watchdog_required +def test_watching_mode_reloads_on_change(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + source, sink = build_source(sources, [path], lambda b: b.change_detection(ChangeDetection.WATCHING)) + assert isinstance(source._change_detector, filedata.Watcher) + assert flag_values(sink.require_snapshot()) == {'flag1': True} + write_file(path, '{"flagValues": {"flag1": false}}') + assert flag_values(sink.require_snapshot()) == {'flag1': False} + + +def test_polling_mode_reloads_on_change(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + source, sink = build_source(sources, [path], lambda b: b.change_detection(ChangeDetection.POLLING).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + assert isinstance(source._change_detector, filedata.Poller) + assert flag_values(sink.require_snapshot()) == {'flag1': True} + write_file(path, '{"flagValues": {"flag1": false}}') + assert flag_values(sink.require_snapshot()) == {'flag1': False} + + +@pytest.mark.parametrize("mode", [ChangeDetection.POLLING, pytest.param(ChangeDetection.WATCHING, marks=watchdog_required)]) +def test_source_retains_last_good_data_across_malformed_edit(tmp_path, sources, caplog, mode): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + with caplog.at_level(logging.ERROR): + _, sink = build_source(sources, [path], lambda b: b.change_detection(mode).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + sink.require_snapshot() + + # A malformed edit produces no snapshot: the previously applied overrides stay in + # effect because the sink is never called. The failure is logged. + write_file(path, '{"flagValues"') + sink.require_no_snapshot(1.5) + deadline = time.time() + TEST_TIMEOUT + while not any('Unable to load flag data' in r.getMessage() for r in caplog.records): + assert time.time() < deadline, "the load failure was not logged" + time.sleep(0.01) + + # Fixing the file recovers, through the change notification or the failure retry. + write_file(path, '{"flagValues": {"flag1": false}}') + assert flag_values(sink.require_snapshot()) == {'flag1': False} + + +def test_source_retries_a_failed_load_without_a_change_signal(tmp_path, sources, caplog): + # The fixed content has the same size as the malformed content and the file keeps its + # modification time, so the poller sees no change. Only the automatic retry can recover. + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + with caplog.at_level(logging.ERROR): + _, sink = build_source(sources, [path], lambda b: b.poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + sink.require_snapshot() + + malformed = '{"flagValues": {"flag1": false}' + fixed = '{"flagValues": {"flag1":false}}' + assert len(malformed) == len(fixed) + write_file(path, malformed) + deadline = time.time() + TEST_TIMEOUT + while not any('Unable to load flag data' in r.getMessage() for r in caplog.records): + assert time.time() < deadline, "the load failure was not logged" + time.sleep(0.01) + observed = os.stat(path) + write_file(path, fixed) + os.utime(path, ns=(observed.st_atime_ns, observed.st_mtime_ns)) + assert os.stat(path).st_size == observed.st_size + assert flag_values(sink.require_snapshot()) == {'flag1': False} + + +def test_source_does_not_supply_snapshots_after_close(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + source, sink = build_source(sources, [path], lambda b: b.poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + sink.require_snapshot() + source.close() + write_file(path, '{"flagValues": {"flag1": false}}') + sink.require_no_snapshot(2.5) + + +def test_source_close_is_idempotent(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{}') + source, _ = build_source(sources, [path]) + source.close() + source.close() + + +def test_source_start_after_close_does_nothing(tmp_path): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + source = FileOverrideSourceBuilder([path]).build(SDKConfig('SDK_KEY')) + source.close() + sink = CapturingSink() + source.start(sink) + sink.require_no_snapshot(0.2) + + +# --------------------------------------------------------------------------- +# Through the client +# --------------------------------------------------------------------------- + +def test_file_overrides_end_to_end(tmp_path): + # An operator writes, edits, and empties an override file. Evaluations follow without any + # client restart, even though the client never obtains data from LaunchDarkly. + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{}') + user = Context.create('user-key') + source = FileOverrideSourceBuilder([path]).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL) + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(source).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + + def eventually(predicate, message: str) -> None: + deadline = time.time() + TEST_TIMEOUT + while not predicate(): + assert time.time() < deadline, message + time.sleep(0.05) + + with LDClient(config, start_wait=0) as client: + # Not initialized and no override present: the default is served. + detail = client.variation_detail('overridden-flag', user, False) + assert detail.value is False + assert detail.reason == {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'} + + # An operator adds an override. The running client picks it up. + write_file(path, '{"flagValues": {"overridden-flag": true}}') + eventually(lambda: client.variation('overridden-flag', user, False) is True, "the override was not picked up") + + # The override changes value. + write_file(path, '{"flagValues": {"overridden-flag": false}}') + + def changed() -> bool: + detail = client.variation_detail('overridden-flag', user, True) + return detail.value is False and detail.reason.get('overrideAffected') is True + + eventually(changed, "the changed override was not picked up") + + # The override is removed. The not-initialized short-circuit returns. + write_file(path, '{}') + eventually(lambda: client.variation_detail('overridden-flag', user, False).reason.get('errorKind') == 'CLIENT_NOT_READY', "the removed override was not picked up") From 210ba0765b0fcbde80b6b7c58adbfb39dc6180b5 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Sat, 3 Oct 2026 00:55:08 +0000 Subject: [PATCH 16/26] fix: Detect changes from the first read and validate the poll interval of the file override source Change detection is set up before the initial load, so an edit made while the files are first read is picked up. Close stops the reloader even when the change detector fails to close. The builder rejects a poll interval that is not a number or not finite. --- .../overrides/file_override_source.py | 31 +++++---- ldclient/integrations/overrides.py | 7 ++ .../integrations/test_file_override_source.py | 65 ++++++++++++++++++- 3 files changed, 88 insertions(+), 15 deletions(-) diff --git a/ldclient/impl/integrations/overrides/file_override_source.py b/ldclient/impl/integrations/overrides/file_override_source.py index f9367701..2d22bb00 100644 --- a/ldclient/impl/integrations/overrides/file_override_source.py +++ b/ldclient/impl/integrations/overrides/file_override_source.py @@ -75,10 +75,19 @@ def apply(merged: MergeResult) -> None: retry_delay=DEFAULT_RETRY_DELAY, skip_unchanged=True, ) + # Change detection is set up before the initial load, so an edit made while the files + # are first read is picked up by the reload it triggers instead of waiting for a later + # change. with self._lock: if self._closed: return self._reloader = reloader + if self._change_detection == ChangeDetection.WATCHING: + self._change_detector = Watcher(self._paths, reloader.trigger) + else: + poller = Poller(self._paths, self._poll_interval, reloader.trigger) + poller.start() + self._change_detector = poller # The initial load runs synchronously, so overrides present in the files are in effect # by the time the client constructor returns. A file that does not exist yet @@ -87,16 +96,6 @@ def apply(merged: MergeResult) -> None: # the file is readable. reloader.reload_now() - with self._lock: - if self._closed: - return - if self._change_detection == ChangeDetection.WATCHING: - self._change_detector = Watcher(self._paths, reloader.trigger) - else: - poller = Poller(self._paths, self._poll_interval, reloader.trigger) - poller.start() - self._change_detector = poller - def close(self) -> None: with self._lock: if self._closed: @@ -106,10 +105,14 @@ def close(self) -> None: reloader = self._reloader self._change_detector = None self._reloader = None - if change_detector is not None: - change_detector.close() - if reloader is not None: - reloader.close() + try: + if change_detector is not None: + change_detector.close() + finally: + # The reloader stops even when the change detector fails to close, so no reload + # can run after close. + if reloader is not None: + reloader.close() def _log_overrides_in_effect(merged: MergeResult) -> None: diff --git a/ldclient/integrations/overrides.py b/ldclient/integrations/overrides.py index 6933253b..e5ca0537 100644 --- a/ldclient/integrations/overrides.py +++ b/ldclient/integrations/overrides.py @@ -26,6 +26,7 @@ counters, so LaunchDarkly can distinguish them. They produce no individual evaluation events. """ +import math from typing import List, Optional, Union from ldclient.config import DataSourceBuilderConfig, OverrideSourceBuilder @@ -126,7 +127,13 @@ def poll_interval(self, seconds: float) -> 'FileOverrideSourceBuilder': :attr:`MINIMUM_POLL_INTERVAL` is raised to the minimum. :param seconds: the interval in seconds + :raises TypeError: when the interval is not a number + :raises ValueError: when the interval is not a finite number """ + if isinstance(seconds, bool) or not isinstance(seconds, (int, float)): + raise TypeError("the poll interval must be a number of seconds") + if not math.isfinite(seconds): + raise ValueError("the poll interval must be a finite number of seconds") self.__poll_interval = seconds return self diff --git a/ldclient/testing/integrations/test_file_override_source.py b/ldclient/testing/integrations/test_file_override_source.py index ab940a85..a3bdbea5 100644 --- a/ldclient/testing/integrations/test_file_override_source.py +++ b/ldclient/testing/integrations/test_file_override_source.py @@ -4,9 +4,10 @@ """ import logging import os +import threading import time from queue import Empty, Queue -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Set, Tuple import pytest @@ -164,6 +165,18 @@ def test_builder_clamps_poll_interval_to_the_minimum(caplog): assert source._poll_interval == 2.5 +@pytest.mark.parametrize('seconds', ['1', None, True], ids=['string', 'none', 'bool']) +def test_builder_rejects_a_poll_interval_that_is_not_a_number(seconds): + with pytest.raises(TypeError): + FileOverrideSourceBuilder(['a']).poll_interval(seconds) + + +@pytest.mark.parametrize('seconds', [float('nan'), float('inf'), float('-inf')], ids=['nan', 'inf', '-inf']) +def test_builder_rejects_a_poll_interval_that_is_not_finite(seconds): + with pytest.raises(ValueError): + FileOverrideSourceBuilder(['a']).poll_interval(seconds) + + def test_builder_rejects_watching_without_watchdog(monkeypatch): import ldclient.integrations.overrides as module monkeypatch.setattr(module, 'have_watchdog', False) @@ -369,6 +382,56 @@ def test_source_retries_a_failed_load_without_a_change_signal(tmp_path, sources, assert flag_values(sink.require_snapshot()) == {'flag1': False} +@pytest.mark.parametrize("mode", [ChangeDetection.POLLING, pytest.param(ChangeDetection.WATCHING, marks=watchdog_required)]) +def test_source_detects_an_edit_made_during_the_initial_load(tmp_path, sources, monkeypatch, mode): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": "initial"}}') + original_reload_now = filedata.Reloader.reload_now + + def load_then_edit(reloader: filedata.Reloader) -> None: + # The edit lands after the files were read and before start returns. + original_reload_now(reloader) + write_file(path, '{"flagValues": {"flag1": "edited-after-the-first-read"}}') + + monkeypatch.setattr(filedata.Reloader, 'reload_now', load_then_edit) + _, sink = build_source(sources, [path], lambda b: b.change_detection(mode).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + + # The initial snapshot holds what the first read saw. The edit is detected and applied + # without any further change to the file. + assert flag_values(sink.require_snapshot()) == {'flag1': 'initial'} + assert flag_values(sink.require_snapshot()) == {'flag1': 'edited-after-the-first-read'} + + +def reloader_threads() -> Set[threading.Thread]: + return {t for t in threading.enumerate() if t.name == 'ldclient.filedata.reloader'} + + +def test_source_close_stops_the_reloader_when_the_change_detector_fails_to_close(tmp_path, sources, monkeypatch): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{}') + before = reloader_threads() + source, sink = build_source(sources, [path]) + sink.require_snapshot() + started = reloader_threads() - before + assert len(started) == 1 + + detector: Any = source._change_detector + original_close = detector.close + + def failing_close() -> None: + original_close() + raise RuntimeError("detector close failure") + + monkeypatch.setattr(detector, 'close', failing_close) + + # The failure propagates, and the reloader is stopped all the same: its worker ends. + with pytest.raises(RuntimeError): + source.close() + worker = started.pop() + worker.join(TEST_TIMEOUT) + assert not worker.is_alive() + + def test_source_does_not_supply_snapshots_after_close(tmp_path, sources): path = os.path.join(str(tmp_path), 'overrides.json') write_file(path, '{"flagValues": {"flag1": true}}') From eb41d277c1e7e8fb2a1dc93a3ed4f1e0ef3b86c9 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:17:05 +0000 Subject: [PATCH 17/26] fix: Reject a poll interval longer than the runtime can wait --- ldclient/integrations/overrides.py | 6 +++++- ldclient/testing/integrations/test_file_override_source.py | 6 ++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/ldclient/integrations/overrides.py b/ldclient/integrations/overrides.py index e5ca0537..2aa1d8ba 100644 --- a/ldclient/integrations/overrides.py +++ b/ldclient/integrations/overrides.py @@ -27,6 +27,7 @@ """ import math +import threading from typing import List, Optional, Union from ldclient.config import DataSourceBuilderConfig, OverrideSourceBuilder @@ -128,12 +129,15 @@ def poll_interval(self, seconds: float) -> 'FileOverrideSourceBuilder': :param seconds: the interval in seconds :raises TypeError: when the interval is not a number - :raises ValueError: when the interval is not a finite number + :raises ValueError: when the interval is not a finite number, or is longer than the + runtime can wait """ if isinstance(seconds, bool) or not isinstance(seconds, (int, float)): raise TypeError("the poll interval must be a number of seconds") if not math.isfinite(seconds): raise ValueError("the poll interval must be a finite number of seconds") + if seconds > threading.TIMEOUT_MAX: + raise ValueError("the poll interval must not exceed %s seconds" % threading.TIMEOUT_MAX) self.__poll_interval = seconds return self diff --git a/ldclient/testing/integrations/test_file_override_source.py b/ldclient/testing/integrations/test_file_override_source.py index a3bdbea5..412ed763 100644 --- a/ldclient/testing/integrations/test_file_override_source.py +++ b/ldclient/testing/integrations/test_file_override_source.py @@ -171,6 +171,12 @@ def test_builder_rejects_a_poll_interval_that_is_not_a_number(seconds): FileOverrideSourceBuilder(['a']).poll_interval(seconds) +def test_builder_rejects_a_poll_interval_longer_than_the_runtime_can_wait(): + with pytest.raises(ValueError) as excinfo: + FileOverrideSourceBuilder(['a']).poll_interval(1e20) + assert 'must not exceed' in str(excinfo.value) + + @pytest.mark.parametrize('seconds', [float('nan'), float('inf'), float('-inf')], ids=['nan', 'inf', '-inf']) def test_builder_rejects_a_poll_interval_that_is_not_finite(seconds): with pytest.raises(ValueError): From a8f4be94c38ba6d09f6f440e91bd273e86fd36a3 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:23:22 +0000 Subject: [PATCH 18/26] fix: Drop a file path listed more than once instead of treating its entries as duplicate keys A path listed more than once would make every entry of that file a duplicate of itself: under the default fail handling the load failed, and under ignore the first listing won by accident. The builder now drops the repeats, keeps the first occurrence in its place in the order, and logs a warning once at build time. This is the behavior the Node SDK has. --- ldclient/integrations/overrides.py | 9 ++++++++- .../testing/integrations/test_file_override_source.py | 10 ++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/ldclient/integrations/overrides.py b/ldclient/integrations/overrides.py index 2aa1d8ba..3d93bc68 100644 --- a/ldclient/integrations/overrides.py +++ b/ldclient/integrations/overrides.py @@ -157,7 +157,14 @@ def build(self, config: DataSourceBuilderConfig) -> OverrideSource: # pylint: d log.warning("Poll interval %s is below the minimum for the file-based override source; using %s", poll_interval, self.MINIMUM_POLL_INTERVAL) poll_interval = self.MINIMUM_POLL_INTERVAL - return _FileOverrideSource(abs_file_paths(self.__paths), self.__duplicate_keys_handling, self.__change_detection, poll_interval) + # A path listed more than once would make every entry of that file a duplicate of itself, + # so the repeats are dropped and the first occurrence keeps its place in the order. + all_paths = abs_file_paths(self.__paths) + paths = list(dict.fromkeys(all_paths)) + if len(paths) != len(all_paths): + log.warning("The file-based override source lists the same file more than once; the repeats are ignored") + + return _FileOverrideSource(paths, self.__duplicate_keys_handling, self.__change_detection, poll_interval) __all__ = ['ChangeDetection', 'DuplicateKeysHandling', 'FileOverrideSourceBuilder'] diff --git a/ldclient/testing/integrations/test_file_override_source.py b/ldclient/testing/integrations/test_file_override_source.py index 412ed763..36e25bce 100644 --- a/ldclient/testing/integrations/test_file_override_source.py +++ b/ldclient/testing/integrations/test_file_override_source.py @@ -125,6 +125,16 @@ def test_builder_resolves_relative_paths(): assert source._paths == [os.path.abspath('relative/overrides.json')] +def test_builder_drops_a_repeated_path_and_warns(caplog): + # Under the default fail handling the same file listed twice would make every one of its keys + # a duplicate of itself. The repeat is ignored and the first occurrence keeps its place. + with caplog.at_level(logging.WARNING): + source = FileOverrideSourceBuilder(['/a.json', '/b.json', '/a.json']).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._paths == [os.path.abspath('/a.json'), os.path.abspath('/b.json')] + assert any('more than once' in r.getMessage() for r in caplog.records) + + def test_builder_polls_by_default(tmp_path): path = os.path.join(str(tmp_path), 'overrides.json') source = FileOverrideSourceBuilder([path]).build(SDKConfig('SDK_KEY')) From f0ba0d7e2a8ff21b7c7893cbd0941db22212aa61 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:11:50 +0000 Subject: [PATCH 19/26] fix: Treat a repeated override file path in different case as a repeat on Windows The builder dropped repeated paths by exact string match, so on Windows a path listed twice in different case survived as two paths, and every key of that file failed as a duplicate of itself under the default fail handling. Paths are now compared with os.path.normcase, which ignores case on Windows and compares exactly elsewhere. The first spelling keeps its place in the order. --- ldclient/integrations/overrides.py | 13 +++++++++++-- .../integrations/test_file_override_source.py | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/ldclient/integrations/overrides.py b/ldclient/integrations/overrides.py index 3d93bc68..c3b4c954 100644 --- a/ldclient/integrations/overrides.py +++ b/ldclient/integrations/overrides.py @@ -27,6 +27,7 @@ """ import math +import os import threading from typing import List, Optional, Union @@ -158,9 +159,17 @@ def build(self, config: DataSourceBuilderConfig) -> OverrideSource: # pylint: d poll_interval = self.MINIMUM_POLL_INTERVAL # A path listed more than once would make every entry of that file a duplicate of itself, - # so the repeats are dropped and the first occurrence keeps its place in the order. + # so the repeats are dropped and the first occurrence keeps its place in the order. Paths + # are compared the way the platform compares file names: without regard to case on + # Windows, exactly elsewhere. all_paths = abs_file_paths(self.__paths) - paths = list(dict.fromkeys(all_paths)) + paths: List[str] = [] + seen = set() + for path in all_paths: + key = os.path.normcase(path) + if key not in seen: + seen.add(key) + paths.append(path) if len(paths) != len(all_paths): log.warning("The file-based override source lists the same file more than once; the repeats are ignored") diff --git a/ldclient/testing/integrations/test_file_override_source.py b/ldclient/testing/integrations/test_file_override_source.py index 36e25bce..5f583508 100644 --- a/ldclient/testing/integrations/test_file_override_source.py +++ b/ldclient/testing/integrations/test_file_override_source.py @@ -3,6 +3,7 @@ absent files, change detection in both modes, failure retention, and its Info log. """ import logging +import ntpath import os import threading import time @@ -135,6 +136,24 @@ def test_builder_drops_a_repeated_path_and_warns(caplog): assert any('more than once' in r.getMessage() for r in caplog.records) +def test_builder_treats_a_path_repeated_in_different_case_as_a_repeat_only_where_file_names_ignore_case(): + source = FileOverrideSourceBuilder(['/a.json', '/A.json']).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + if os.name == 'nt': + assert source._paths == [os.path.abspath('/a.json')] + else: + assert source._paths == [os.path.abspath('/a.json'), os.path.abspath('/A.json')] + + +def test_builder_drops_a_path_repeated_in_different_case_on_windows(monkeypatch): + # Windows file names ignore case, so two spellings of one path name the same file. The path + # comparison of Windows is used here on any platform. + monkeypatch.setattr(os.path, 'normcase', ntpath.normcase) + source = FileOverrideSourceBuilder(['/a.json', '/b.json', '/A.json']).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._paths == [os.path.abspath('/a.json'), os.path.abspath('/b.json')] + + def test_builder_polls_by_default(tmp_path): path = os.path.join(str(tmp_path), 'overrides.json') source = FileOverrideSourceBuilder([path]).build(SDKConfig('SDK_KEY')) From 0415ec1c8952702c6edd7cdd0a0638484387d25c Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:53:19 +0000 Subject: [PATCH 20/26] docs: Note that a flagValues override is not available to client-side SDKs A flagValues entry becomes a flag without client-side availability, so an all-flags state for client-side SDKs leaves it out even when the flag it overrides is client-side. A full definition with clientSide is the way to override such a flag. A test pins both. --- ldclient/integrations/overrides.py | 6 +++++- .../integrations/test_file_override_source.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/ldclient/integrations/overrides.py b/ldclient/integrations/overrides.py index c3b4c954..782a7397 100644 --- a/ldclient/integrations/overrides.py +++ b/ldclient/integrations/overrides.py @@ -54,7 +54,11 @@ class FileOverrideSourceBuilder(OverrideSourceBuilder): as the files change. The files use the same document format as the file data source: each file is a JSON or YAML document with optional ``flags``, ``flagValues``, and ``segments`` members. ``flagValues`` entries are expanded into full flag definitions that - return the given value for every context. YAML requires the ``pyyaml`` package. + return the given value for every context. A ``flagValues`` entry is not available to + client-side SDKs, so :meth:`ldclient.client.LDClient.all_flags_state` with + ``client_side_only=True`` leaves it out, even when the flag it overrides is available to them. + To override such a flag, define it in ``flags`` with ``"clientSide": true``. YAML requires the + ``pyyaml`` package. When multiple files are configured, their entries are combined in the configured order. The order determines which file wins under the duplicate keys handling. A reload replaces diff --git a/ldclient/testing/integrations/test_file_override_source.py b/ldclient/testing/integrations/test_file_override_source.py index 5f583508..a9aea905 100644 --- a/ldclient/testing/integrations/test_file_override_source.py +++ b/ldclient/testing/integrations/test_file_override_source.py @@ -537,3 +537,22 @@ def changed() -> bool: # The override is removed. The not-initialized short-circuit returns. write_file(path, '{}') eventually(lambda: client.variation_detail('overridden-flag', user, False).reason.get('errorKind') == 'CLIENT_NOT_READY', "the removed override was not picked up") + + +def test_client_side_all_flags_state_includes_a_full_definition_with_client_side_but_not_a_value_entry(tmp_path): + # A flagValues entry is not available to client-side SDKs. A full definition that sets + # clientSide is, so that is how an operator overrides a flag that client-side SDKs use. + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"value-flag": true}, ' + '"flags": {"full-flag": {"key": "full-flag", "version": 1, "on": true, "variations": [true], ' + '"fallthrough": {"variation": 0}, "clientSide": true}}}') + source = FileOverrideSourceBuilder([path]) + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(source).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + + with LDClient(config, start_wait=0) as client: + user = Context.create('user-key') + assert set(client.all_flags_state(user).to_values_map().keys()) == {'value-flag', 'full-flag'} + state = client.all_flags_state(user, client_side_only=True) + assert state.valid + assert state.to_values_map() == {'full-flag': True} From 0d55468619ae021ef0e1e7b9792367a8e5c0375a Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:34:45 -0700 Subject: [PATCH 21/26] feat: Add flag overrides to the contract test services Both contract test services advertise the flag-overrides capability and accept the overrides object in the SDK configuration: filePaths, and the optional duplicateKeysHandling, changeDetection, and pollIntervalMs properties, which configure the file-based override source on the FDv2 data system. An overrides object without a data system configuration is rejected, because overrides are an option of the FDv2 data system only. --- contract-tests/async_client_entity.py | 28 ++++++++++++++++++++++----- contract-tests/async_service.py | 1 + contract-tests/client_entity.py | 19 ++++++++++++++++++ contract-tests/service.py | 1 + 4 files changed, 44 insertions(+), 5 deletions(-) diff --git a/contract-tests/async_client_entity.py b/contract-tests/async_client_entity.py index 2569a156..d45f9072 100644 --- a/contract-tests/async_client_entity.py +++ b/contract-tests/async_client_entity.py @@ -26,6 +26,7 @@ ) from ldclient.impl.util import Result from ldclient.integrations import Redis +from ldclient.integrations.overrides import FileOverrideSourceBuilder from ldclient.interfaces import DataStoreMode from ldclient.migrations import ( AsyncMigratorBuilder, @@ -56,8 +57,10 @@ async def start(self): } datasystem_config = config_params.get('dataSystem') + if config_params.get('overrides') is not None and datasystem_config is None: + raise ValueError("flag overrides require the data system to be configured") if datasystem_config is not None: - opts["datasystem_config"] = _build_async_data_system(datasystem_config, opts) + opts["datasystem_config"] = _build_async_data_system(datasystem_config, opts, config_params.get('overrides')) elif config_params.get("streaming") is not None: streaming = config_params["streaming"] if streaming.get("baseUri") is not None: @@ -297,13 +300,13 @@ def _set_optional_value(params_in: dict, name_in: str, func: Callable[[Any], Any func(params_in[name_in]) -def _build_async_data_system(datasystem_config: dict, opts: dict) -> AsyncDataSystemConfig: +def _build_async_data_system(datasystem_config: dict, opts: dict, overrides_config: Optional[dict] = None) -> AsyncDataSystemConfig: """Build an AsyncDataSystemConfig from the harness's dataSystem config. Wires the FDv2 initializers, the ordered synchronizer chain, the FDv1 - fallback synchronizer, the payload filter, and an optional async - persistent store. The async client injects its shared aiohttp session - into these builders when it starts. + fallback synchronizer, the payload filter, an optional async persistent + store, and the optional file-based override source. The async client + injects its shared aiohttp session into these builders when it starts. """ initializers: Optional[list] = None init_configs = datasystem_config.get('initializers') @@ -356,6 +359,9 @@ def _build_async_data_system(datasystem_config: dict, opts: dict) -> AsyncDataSy "fdv1_fallback_synchronizer": fdv1_fallback_synchronizer, } + if overrides_config is not None: + ds_kwargs["override_source"] = _make_override_source(overrides_config) + store_config = datasystem_config.get("store") if store_config is not None: persistent_store_config = store_config.get("persistentDataStore") @@ -370,6 +376,18 @@ def _build_async_data_system(datasystem_config: dict, opts: dict) -> AsyncDataSy return AsyncDataSystemConfig(**ds_kwargs) +def _make_override_source(params: dict) -> FileOverrideSourceBuilder: + """ + Builds the file-based override source from the harness's overrides configuration. The + files are written by the harness on a file system shared with this service. + """ + builder = FileOverrideSourceBuilder(params["filePaths"]) + _set_optional_value(params, "duplicateKeysHandling", builder.duplicate_keys_handling) + _set_optional_value(params, "changeDetection", builder.change_detection) + _set_optional_time(params, "pollIntervalMs", builder.poll_interval) + return builder + + def _create_async_persistent_store(persistent_store_config: dict): """Create an async persistent feature store from the harness config. diff --git a/contract-tests/async_service.py b/contract-tests/async_service.py index b3c04d8a..6a11abc0 100644 --- a/contract-tests/async_service.py +++ b/contract-tests/async_service.py @@ -69,6 +69,7 @@ async def handle_status(request: aiohttp.web.Request) -> aiohttp.web.Response: 'migrations', 'persistent-data-store-redis', 'fdv1-fallback', + 'flag-overrides', ] } return aiohttp.web.Response( diff --git a/contract-tests/client_entity.py b/contract-tests/client_entity.py index f93b5d85..c02fc792 100644 --- a/contract-tests/client_entity.py +++ b/contract-tests/client_entity.py @@ -28,6 +28,7 @@ from ldclient.feature_store import CacheConfig from ldclient.impl.datasourcev2.polling import PollingDataSourceBuilder from ldclient.integrations import Consul, DynamoDB, Redis +from ldclient.integrations.overrides import FileOverrideSourceBuilder from ldclient.interfaces import DataStoreMode @@ -44,6 +45,8 @@ def __init__(self, tag, config): } datasystem_config = config.get('dataSystem') + if config.get('overrides') is not None and datasystem_config is None: + raise ValueError("flag overrides require the data system to be configured") if datasystem_config is not None: datasystem = custom() @@ -107,6 +110,10 @@ def __init__(self, tag, config): datasystem.data_store(store, store_mode) + overrides_config = config.get('overrides') + if overrides_config is not None: + datasystem.overrides(_make_override_source(overrides_config)) + opts["datasystem_config"] = datasystem.build() elif config.get("streaming") is not None: @@ -325,6 +332,18 @@ def _set_optional_value(params_in: dict, name_in: str, func: Callable[[Any], Any func(params_in[name_in]) +def _make_override_source(params: dict) -> FileOverrideSourceBuilder: + """ + Builds the file-based override source from the harness's overrides configuration. The + files are written by the harness on a file system shared with this service. + """ + builder = FileOverrideSourceBuilder(params["filePaths"]) + _set_optional_value(params, "duplicateKeysHandling", builder.duplicate_keys_handling) + _set_optional_value(params, "changeDetection", builder.change_detection) + _set_optional_time(params, "pollIntervalMs", builder.poll_interval) + return builder + + def _create_persistent_store(persistent_store_config: dict): """ Creates a persistent store instance based on the configuration. diff --git a/contract-tests/service.py b/contract-tests/service.py index a8e93674..2583748e 100644 --- a/contract-tests/service.py +++ b/contract-tests/service.py @@ -86,6 +86,7 @@ def status(): 'flag-change-listeners', 'flag-value-change-listeners', 'fdv1-fallback', + 'flag-overrides', ] } return json.dumps(body), 200, {'Content-type': 'application/json'} From 698f9ed08bcd5cfa2a463f428663a40b28d603c3 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Mon, 2 Nov 2026 23:34:01 +0000 Subject: [PATCH 22/26] fix: Keep overrides through an in-place rewrite of their files A writer that rewrites a file in place truncates it and then writes the new content. With pyyaml installed, a load in between parsed the empty file as a document with no entries whatever its name, so the override source briefly removed every override. The parser is now chosen by the file name extension: .yaml and .yml are YAML, and any other file is JSON. An empty JSON file is not a document and fails to load; an empty YAML file has no entries. A YAML file without pyyaml is a configuration error. A load that finds a file with no entries (empty, or YAML that parses to nothing) while that file's content is in effect holds it back: the current data stays, nothing is reported, and the retry is armed for when the file has stayed so, unmodified, for the retry delay. A change of the file's modification time or size, read through symbolic links, starts the delay over, and loads stop holding files back after five retry delays in a row. --- ldclient/impl/integrations/files/filedata.py | 406 +++++++-- .../overrides/file_override_source.py | 4 +- ldclient/integrations/overrides.py | 34 +- ldclient/testing/file_util.py | 26 + .../integrations/test_file_override_source.py | 174 ++++ .../testing/integrations/test_filedata.py | 818 +++++++++++++++++- 6 files changed, 1330 insertions(+), 132 deletions(-) create mode 100644 ldclient/testing/file_util.py diff --git a/ldclient/impl/integrations/files/filedata.py b/ldclient/impl/integrations/files/filedata.py index a4f71d6a..41a36ff7 100644 --- a/ldclient/impl/integrations/files/filedata.py +++ b/ldclient/impl/integrations/files/filedata.py @@ -6,9 +6,11 @@ A data file is a JSON or YAML document with optional ``flags``, ``flagValues``, and ``segments`` members. ``flags`` and ``segments`` hold full definitions keyed by key. ``flagValues`` maps a flag key to a single value. It expands into a full flag definition -that returns that value for every context. +that returns that value for every context. The file name extension decides the format: a +file whose name ends in ``.yaml`` or ``.yml`` is YAML, and any other file is JSON. """ +import codecs import hashlib import json import os @@ -54,6 +56,9 @@ # because it does not exist yet. _WATCH_RETRY_INTERVAL = 1.0 +# The file name extensions of YAML files, compared without regard to case. Any other file is JSON. +YAML_EXTENSIONS = frozenset([".yaml", ".yml"]) + # The watchdog event types that can change a file's content or presence. Opening or reading a # file also produces events, and a reload reads the files, so those must not count as changes. _CHANGE_EVENT_TYPES = frozenset(["created", "modified", "moved", "deleted", "closed"]) @@ -148,33 +153,66 @@ def make_flag_with_value(key: str, value: Any) -> FeatureFlag: return FeatureFlag({"key": key, "version": 1, "on": True, "fallthrough": {"variation": 0}, "variations": [value]}) +def is_yaml_path(path: str) -> bool: + """ + Whether a data file is a YAML document. The file name extension decides: ``.yaml`` and + ``.yml``, in any case, are YAML, and any other file, including one with no extension, is JSON. + """ + return os.path.splitext(path)[1].lower() in YAML_EXTENSIONS + + def read_file(path: str) -> Document: - """Reads and parses a single data file, which may be in JSON or YAML format.""" + """Reads and parses a single data file, in the format its file name extension selects.""" try: with open(path, "rb") as f: raw = f.read() except OSError as e: raise FileReadError(path, "unable to read file: %s" % e) try: - return parse_document(raw) + return parse_document(raw, is_yaml_path(path)) except Exception as e: raise FileReadError(path, "error parsing file: %s" % e) -def parse_document(raw: bytes) -> Document: +def has_content(raw: bytes) -> bool: + """Whether the raw content of a file holds anything other than white space and a byte order mark.""" + return raw.removeprefix(codecs.BOM_UTF8).strip() != b"" + + +def parse_document(raw: bytes, is_yaml: bool = False) -> Document: + """ + Parses the raw content of a data file as YAML when ``is_yaml`` is true, which requires the + ``pyyaml`` package, and as JSON otherwise. The file name extension decides the format; see + :func:`is_yaml_path`. A byte order mark at the start is ignored. + + Whether a file with no content is valid depends on its format, like the rest of its content. + A JSON file with no content, or only white space, is not a JSON document, so it fails to + parse. A YAML file with no content, or with only comments or a document marker, is a + document with no entries. + """ + document = _parse_content(raw, is_yaml) + return Document() if document is None else document + + +def _parse_content(raw: bytes, is_yaml: bool) -> Optional[Document]: """ - Parses the raw content of a data file, which may be JSON or YAML. The parser is chosen the - way the file data sources choose it: the YAML parser reads both formats when the ``pyyaml`` - package is installed, and the standard JSON parser is used when it is not. An empty - document is valid and holds no entries. + The parse behind :func:`parse_document`. Returns None when the parser reads the content as + nothing, as YAML parsers do for a file with no content or only comments, rather than as a + document, even one with no members such as ``{}``. """ - text = raw.decode("utf-8") - if have_yaml: - parsed = yaml.safe_load(text) # pyyaml correctly parses JSON too + text = raw.decode("utf-8-sig") + if is_yaml: + if not have_yaml: + raise ValueError("the pyyaml package is required to read YAML") + parsed = yaml.safe_load(text) else: + if not has_content(raw): + # Not a JSON document. A file that is rewritten in place can be empty for a moment, + # and the failure keeps the last good data until the file is complete. + raise ValueError("the file has no content, which is not a JSON document") parsed = json.loads(text) if parsed is None: - return Document() + return None if not isinstance(parsed, dict): raise ValueError("the document must be an object") @@ -255,38 +293,71 @@ def load_files(paths: List[str], duplicate_keys_handling: DuplicateKeysHandling, and the handling is FAIL. When ``skip_missing_paths`` is true, a file that does not exist contributes no entries instead of failing the load. """ - merged, _ = _load_files_hashed(paths, duplicate_keys_handling, skip_missing_paths) + merged, _ = _parse_files(_read_files(paths, skip_missing_paths), duplicate_keys_handling) return merged -def _load_files_hashed(paths: List[str], duplicate_keys_handling: DuplicateKeysHandling, skip_missing_paths: bool) -> Tuple[MergeResult, bytes]: +@dataclass +class _FileRead: + """What one load read from one configured file.""" + + path: str + raw: Optional[bytes] + """The raw content, or None when the file does not exist and missing files are skipped.""" + + +def _read_files(paths: List[str], skip_missing_paths: bool) -> List[_FileRead]: """ - The load behind :func:`load_files`. It also returns a digest of the raw file contents, so a - caller can tell whether a later load read the same bytes. One read feeds both the digest - and the parse, so the digest can never disagree with the content that was parsed. + Reads the raw content of all of the given files in order. Raises :class:`FileReadError` + when a file cannot be read, including a missing file unless ``skip_missing_paths`` is true. """ - documents: List[Document] = [] - files: List[FileSummary] = [] - hasher = hashlib.sha256() + contents: List[_FileRead] = [] for path in paths: try: with open(path, "rb") as f: - raw = f.read() + contents.append(_FileRead(path, f.read())) except FileNotFoundError: if skip_missing_paths: log.debug("File %s does not exist; it contributes no data", path) - files.append(FileSummary(path=path)) + contents.append(_FileRead(path, None)) continue raise FileReadError(path, "unable to read file: the file does not exist") except OSError as e: raise FileReadError(path, "unable to read file: %s" % e) - hasher.update(raw) + return contents + + +def _parse_file(path: str, raw: bytes) -> Optional[Document]: + """Parses what was read from a file that exists. Returns None when the parser read nothing.""" + try: + return _parse_content(raw, is_yaml_path(path)) + except Exception as e: + raise FileReadError(path, "error parsing file: %s" % e) + + +def _parse_files(contents: List[_FileRead], duplicate_keys_handling: DuplicateKeysHandling) -> Tuple[MergeResult, bytes]: + """Parses and merges the content that :func:`_read_files` read.""" + return _merge_files([(read, None if read.raw is None else _parse_file(read.path, read.raw)) for read in contents], duplicate_keys_handling) + + +def _merge_files(parsed: List[Tuple[_FileRead, Optional[Document]]], duplicate_keys_handling: DuplicateKeysHandling) -> Tuple[MergeResult, bytes]: + """ + Merges the parsed files. It also returns a digest of the raw file contents, so a caller can + tell whether a later load read the same bytes. One read feeds both the digest and the parse, + so the digest can never disagree with the content that was parsed. The digest covers the + bytes of the files that exist, not which file holds them. + """ + documents: List[Document] = [] + files: List[FileSummary] = [] + hasher = hashlib.sha256() + for read, document in parsed: + if read.raw is None: + files.append(FileSummary(path=read.path)) + continue + hasher.update(read.raw) hasher.update(b"\0") - try: - documents.append(parse_document(raw)) - except Exception as e: - raise FileReadError(path, "error parsing file: %s" % e) - files.append(FileSummary(path=path, present=True)) + documents.append(Document() if document is None else document) + files.append(FileSummary(path=read.path, present=True)) merged = merge(documents, duplicate_keys_handling) # The documents are the present files in order. Copy their counts onto the file summaries. @@ -300,12 +371,60 @@ def _load_files_hashed(paths: List[str], duplicate_keys_handling: DuplicateKeysH return merged, hasher.digest() +FileState = Optional[Tuple[int, int]] + + +def _observe(path: str) -> FileState: + """ + Observes the state of a file: its modification time and size, through any symbolic link, or + None when it does not exist or cannot be examined. + """ + try: + info = os.stat(path) + return (info.st_mtime_ns, info.st_size) + except OSError: + return None + + +def _observe_all(paths: List[str]) -> List[FileState]: + """Observes the state of each file, as :func:`_observe` does.""" + return [_observe(path) for path in paths] + + +# Loads stop holding files back once they have held some back for this many retry delays in a +# row. +_MAX_HOLD_RETRY_DELAYS = 5 + + +@dataclass +class _EmptyFile: + """A file whose content is in effect and that loads have found with no entries since.""" + + since: float + """Since when the file has had no entries without being written, on the reloader's clock.""" + state: FileState + """The file's state then: its modification time and size, through any symbolic link.""" + + class Reloader: """ Owns the reload cycle for a set of data files. It serializes reloads, debounces change signals, retains the last good result on failure by not calling ``apply``, retries after failures, and skips applications that would change nothing. + A file can be rewritten in place: a writer truncates it and then writes the new content, and + a load in between finds it empty. When retries are enabled, a load that finds a file with no + entries, because it is empty or holds only what its parser reads as nothing (a YAML comment + or document marker), while that file's content is in effect, holds the file back: it keeps + the current data without reporting a failure, and the other files' changes wait too, since a + load applies all files or none. The file is held back until it has stayed so, without being + written, for the retry delay; a change of its modification time or size starts the delay + over. Loads stop holding files back once they have held some back for five retry delays in a + row, so files that keep being rewritten are applied in the end. A file whose content is not + in effect, such as a placeholder that has always been empty, is never held back. Every file + is read and parsed before any is held back, so files emptied together wait together, and a + file that cannot be parsed fails the load even while another is held back. + The worker thread starts on the first :meth:`reload_now` or :meth:`trigger` call, so a reloader that is constructed but never used does not leak a thread. """ @@ -320,6 +439,7 @@ def __init__( debounce_delay: float = 0.0, retry_delay: float = 0.0, skip_unchanged: bool = False, + clock: Callable[[], float] = time.monotonic, ): """ :param paths: the files to load, in order. The order determines which file wins under @@ -336,9 +456,13 @@ def __init__( :param debounce_delay: how long to wait after a trigger for further triggers to settle before reloading. Zero reloads on every trigger. :param retry_delay: how long to wait after a failed reload before retrying it - automatically. Zero disables the automatic retry. + automatically, and how long a file whose content is in effect must have no entries, + without being written, before that is applied. Zero disables the automatic retry, and + an empty file is then parsed as it is. :param skip_unchanged: when true, a load whose raw file contents are identical to the last applied contents does not call ``apply`` + :param clock: the monotonic clock, in seconds, that times how long files are held back. + Tests replace it; the timers always use ``time.monotonic``. """ self._paths = list(paths) self._duplicate_keys_handling = duplicate_keys_handling @@ -348,6 +472,7 @@ def __init__( self._debounce_delay = debounce_delay self._retry_delay = retry_delay self._skip_unchanged = skip_unchanged + self._clock = clock # Guards the deadlines, the closed flag, and the worker start. self._cond = threading.Condition() @@ -356,10 +481,22 @@ def __init__( self._debounce_deadline: Optional[float] = None self._retry_deadline: Optional[float] = None - # Serializes the load work between reload_now and the worker thread. + # Serializes the load work between reload_now and the worker thread. The fields below are + # only touched while holding it. self._reload_lock = threading.Lock() self._last_good_digest: Optional[bytes] = None self._last_error_message: Optional[str] = None + # Whether each configured file contributed content to the data in effect. A missing file, + # or one with no entries, contributed none. + self._applied_had_content: Dict[str, bool] = {} + # The files whose content is in effect and that loads have found with no entries since. + # A file is removed when a load reads it with entries. + self._empty_files: Dict[str, _EmptyFile] = {} + # When the current run of loads that hold files back began, or None when the last load + # held none back. + self._holding_since: Optional[float] = None + # Set by a load that holds files back. + self._held_back = False def reload_now(self) -> None: """ @@ -367,11 +504,14 @@ def reload_now(self) -> None: arms the same automatic retry as a failed triggered reload. """ self._ensure_started() - if not self._reload() and self._retry_delay > 0: + retry_after = self._reload() + if retry_after is not None and self._retry_delay > 0: with self._cond: - if not self._closed and self._retry_deadline is None: - self._retry_deadline = time.monotonic() + self._retry_delay - self._cond.notify() + if not self._closed: + deadline = time.monotonic() + retry_after + if self._retry_deadline is None or deadline < self._retry_deadline: + self._retry_deadline = deadline + self._cond.notify() def trigger(self) -> None: """ @@ -421,7 +561,7 @@ def _run(self) -> None: continue if self._debounce_deadline is not None and self._debounce_deadline <= now: # A triggered reload supersedes a pending retry. It either succeeds, or - # it fails and arms a fresh retry below. + # it fails or holds files back and arms a fresh retry below. self._debounce_deadline = None self._retry_deadline = None is_retry = False @@ -431,68 +571,170 @@ def _run(self) -> None: break if is_retry: - log.debug("Retrying flag data load after earlier failure") + log.debug("Loading flag data again after an earlier failed or held back load") else: log.info("Reloading flag data after detecting a change") try: - ok = self._reload() + retry_after = self._reload() except Exception as e: log.exception("Unexpected error while reloading flag data: %s", e) - ok = True - if not ok and self._retry_delay > 0: + retry_after = None + if retry_after is not None and self._retry_delay > 0: with self._cond: if not self._closed: - self._retry_deadline = time.monotonic() + self._retry_delay + self._retry_deadline = time.monotonic() + retry_after def _is_closed(self) -> bool: with self._cond: return self._closed - def _reload(self) -> bool: + def _reload(self) -> Optional[float]: """ - Performs one full load of all configured files. Returns whether the load succeeded, - which decides whether a retry is armed. A skipped no-op application counts as success. + Performs one full load of all configured files. Returns None when the load succeeded, + including a skipped no-op application, or the time after which to load again: the retry + delay after a failure, or the time a hold lasts. """ with self._reload_lock: if self._is_closed(): - return True - try: - merged, digest = _load_files_hashed(self._paths, self._duplicate_keys_handling, self._skip_missing_paths) - except Exception as e: - return self._fail(e) + return None + self._held_back = False + retry_after = self._load() + if not self._held_back: + # A run of loads that hold files back ends with any load that does not. + self._holding_since = None + return retry_after + + def _load(self) -> Optional[float]: + try: + contents = _read_files(self._paths, self._skip_missing_paths) + except Exception as e: + return self._fail(e) + + # Every file is parsed and classified before any is held back, so that every file that + # is found with no entries starts its delay in this load. + now = self._clock() + held_back: List[str] = [] + parsed: List[Tuple[_FileRead, Optional[Document]]] = [] + had_content: Dict[str, bool] = {} + for read in contents: + if read.raw is None: + had_content[read.path] = False + parsed.append((read, None)) + continue + # A file with no content, or with content that its parser reads as nothing, such as + # YAML with only a comment, has no document and so no entries. A read can see that + # much of a file that is being written. Content with no document is invalid JSON, so + # it is only parsed when the file is not held back. {} is a document. + document: Optional[Document] = None + has_document = has_content(read.raw) + if has_document: + try: + document = _parse_file(read.path, read.raw) + except FileReadError as e: + return self._fail(e) + has_document = document is not None + if has_document: + self._empty_files.pop(read.path, None) + elif self._should_hold_back(read.path, now): + held_back.append(read.path) + continue + elif not has_content(read.raw): + try: + document = _parse_file(read.path, read.raw) + except FileReadError as e: + return self._fail(e) + had_content[read.path] = has_document + parsed.append((read, document)) + if len(held_back) > 0: + return self._hold_back(held_back, now) - # A close may have happened while the files were being read. Deliver nothing then. - if self._is_closed(): - return True - - # A success right after a failure applies even when the content is unchanged since - # the last success. The consumer heard about the failure and only an application - # tells it that things are good again. - recovering = self._last_error_message is not None - if self._skip_unchanged and not recovering and digest == self._last_good_digest: - return True - - # Nothing is remembered until the consumer has accepted the result. A result the - # consumer rejects must not become the baseline that skip-unchanged compares - # against, and must not count as a recovery. - try: - self._apply(merged) - except Exception as e: - return self._fail(e) - self._last_error_message = None - self._last_good_digest = digest - return True + try: + merged, digest = _merge_files(parsed, self._duplicate_keys_handling) + except Exception as e: + return self._fail(e) + + # A close may have happened while the files were being read. Deliver nothing then. + if self._is_closed(): + return None + + # A success right after a failure applies even when the content is unchanged since the + # last success. The consumer heard about the failure and only an application tells it + # that things are good again. + recovering = self._last_error_message is not None + if self._skip_unchanged and not recovering and digest == self._last_good_digest: + # The digest does not say which file holds which bytes, so content that moved from + # one file to another is skipped as unchanged, and what each file contributes has to + # be remembered here too. + self._applied_had_content.update(had_content) + return None + + # Nothing is remembered until the consumer has accepted the result. A result the + # consumer rejects must not become the baseline that skip-unchanged compares against, + # and must not count as a recovery. + try: + self._apply(merged) + except Exception as e: + return self._fail(e) + self._last_error_message = None + self._last_good_digest = digest + self._applied_had_content.update(had_content) + return None - def _fail(self, err: Exception) -> bool: + def _should_hold_back(self, path: str, now: float) -> bool: + """ + Decides whether a file that a load found with no entries is held back. A file whose + content is in effect and that has none now may be in the middle of an in-place rewrite. + It is held back until it has stayed so, without being written, for the retry delay. A + change of its state, its modification time and size read through any symbolic link, + means that it was written since a load last found it so, for example rewritten and + truncated again between two loads, so the delay starts over. Loads stop holding files + back once they have held some back for five retry delays in a row, so that a file that + keeps being written while every load finds it empty, or files that are rewritten in turn, + are not held back without end. A file whose content is not in effect is parsed right + away, and so is any file when retries are off, since nothing would read it again. + """ + if self._retry_delay <= 0 or not self._applied_had_content.get(path, False): + return False + state = _observe(path) + empty = self._empty_files.get(path) + if empty is None: + empty = _EmptyFile(since=now, state=state) + self._empty_files[path] = empty + elif empty.state != state: + empty.since = now + empty.state = state + holding_since = now if self._holding_since is None else self._holding_since + return now - empty.since < self._retry_delay and now - holding_since < _MAX_HOLD_RETRY_DELAYS * self._retry_delay + + def _hold_back(self, held_back: List[str], now: float) -> Optional[float]: + """ + Keeps the current data for a load that holds files back. Whatever the files' format, + nothing is applied or reported. The returned time arms the retry for when the first of + the files has stayed so for the retry delay, or when the loads have held files back for + as long as they may, plus a tenth of the retry delay, so that a timer that fires a little + early still finds the delay over. The change signal for a write starts a load sooner. + """ if self._is_closed(): - return True + return None + if self._holding_since is None: + self._holding_since = now + until = self._holding_since + _MAX_HOLD_RETRY_DELAYS * self._retry_delay + for path in held_back: + log.debug("File %s has no entries, possibly while it is being rewritten; waiting for it to be written before applying it", path) + until = min(until, self._empty_files[path].since + self._retry_delay) + self._held_back = True + return max(until - now, 0.0) + self._retry_delay / 10 + + def _fail(self, err: Exception) -> Optional[float]: + if self._is_closed(): + return None # With automatic retries, a persistent failure would repeat the same log entry and the # same callback on every attempt. Repeats of an identical failure are logged at debug # level and do not call on_error again. message = str(err) if message == self._last_error_message: log.debug("Unable to load flag data: %s", err) - return False + return self._retry_delay self._last_error_message = message log.error("Unable to load flag data: %s", err) if self._on_error is not None: @@ -501,25 +743,7 @@ def _fail(self, err: Exception) -> bool: except Exception as e: # A consumer that raises while it reports a failure must not stop the retry. log.error("Error while reporting a flag data load failure: %s", e) - return False - - -FileState = Optional[Tuple[int, int]] - - -def _observe_all(paths: List[str]) -> List[FileState]: - """ - Observes the state of each file: its modification time and size, or None when it does not - exist or cannot be examined. - """ - states: List[FileState] = [] - for path in paths: - try: - info = os.stat(path) - states.append((info.st_mtime_ns, info.st_size)) - except OSError: - states.append(None) - return states + return self._retry_delay class Poller: diff --git a/ldclient/impl/integrations/overrides/file_override_source.py b/ldclient/impl/integrations/overrides/file_override_source.py index 2d22bb00..220a73b8 100644 --- a/ldclient/impl/integrations/overrides/file_override_source.py +++ b/ldclient/impl/integrations/overrides/file_override_source.py @@ -48,7 +48,9 @@ class _FileOverrideSource(OverrideSource): Reads overrides from the configured files and supplies each successful load to the sink as a full snapshot. A configured file that does not exist contributes no overrides. A file that cannot be read or parsed fails that load, the last good snapshot stays in effect, and - the load is retried. + the load is retried. A file whose content is in effect and that is found with no entries is + held back until it has stayed so for the retry delay, because it may be in the middle of an + in-place rewrite. """ def __init__(self, paths: List[str], duplicate_keys_handling: DuplicateKeysHandling, change_detection: ChangeDetection, poll_interval: float): diff --git a/ldclient/integrations/overrides.py b/ldclient/integrations/overrides.py index 782a7397..03ecbee4 100644 --- a/ldclient/integrations/overrides.py +++ b/ldclient/integrations/overrides.py @@ -35,7 +35,9 @@ from ldclient.impl.integrations.files.filedata import ( DuplicateKeysHandling, abs_file_paths, - have_watchdog + have_watchdog, + have_yaml, + is_yaml_path ) from ldclient.impl.integrations.overrides.file_override_source import ( ChangeDetection, @@ -57,8 +59,13 @@ class FileOverrideSourceBuilder(OverrideSourceBuilder): return the given value for every context. A ``flagValues`` entry is not available to client-side SDKs, so :meth:`ldclient.client.LDClient.all_flags_state` with ``client_side_only=True`` leaves it out, even when the flag it overrides is available to them. - To override such a flag, define it in ``flags`` with ``"clientSide": true``. YAML requires the - ``pyyaml`` package. + To override such a flag, define it in ``flags`` with ``"clientSide": true``. + + The file name extension decides the format of each file: a file whose name ends in + ``.yaml`` or ``.yml``, in any case, is YAML, and any other file is JSON. YAML requires the + ``pyyaml`` package. Whether a file with no content is valid depends on its format: a JSON file + with no content, or only white space, is not a JSON document and fails to load, and a YAML + file with no content is a document with no entries. When multiple files are configured, their entries are combined in the configured order. The order determines which file wins under the duplicate keys handling. A reload replaces @@ -69,6 +76,18 @@ class FileOverrideSourceBuilder(OverrideSourceBuilder): source logs the failure, retries after a short delay, and recovers on its own once the file is readable again. + A file can be rewritten in place. To remove a file's overrides, delete it or write a document + with no entries, such as ``{}``. A JSON file with no content is not a valid document: it fails + to load, and the overrides in effect stay. A file whose content is in effect and that is found + empty, or with only a YAML comment or document marker, may be in the middle of a rewrite. The + source then keeps the overrides in effect, and changes to the other files wait too, since a + reload applies all files or none, until the file has stayed so without being written for + about a second, and for at most about five seconds while files keep being rewritten. After + that, a YAML file is applied with no entries, and an empty JSON file fails to load and keeps + the overrides in effect. YAML that is cut off between two entries can still parse, so for a + YAML file, replacing its entire contents in one operation, such as renaming a fully written + temporary file over it, avoids briefly applying only a part of it. + Whenever the set of overrides in effect changes, including at startup, the source logs the overrides in effect and what each configured file supplied, at Info level. @@ -149,13 +168,18 @@ def poll_interval(self, seconds: float) -> 'FileOverrideSourceBuilder': def build(self, config: DataSourceBuilderConfig) -> OverrideSource: # pylint: disable=unused-argument """ Builds the override source. This is called internally by the SDK. It raises - ``ValueError`` when no file paths were specified or when watching mode was selected - without the ``watchdog`` package. + ``ValueError`` when no file paths were specified, when watching mode was selected + without the ``watchdog`` package, or when a YAML file was specified without the + ``pyyaml`` package. """ if len(self.__paths) == 0: raise ValueError("no file paths were specified for the file-based override source") if self.__change_detection == ChangeDetection.WATCHING and not have_watchdog: raise ValueError("the file-based override source cannot watch files for changes because the watchdog package is not installed; install it or use polling") + if not have_yaml: + for path in self.__paths: + if is_yaml_path(path): + raise ValueError("the file-based override source cannot read the YAML file %s because the pyyaml package is not installed; install it or use JSON" % path) poll_interval = self.__poll_interval if self.__change_detection == ChangeDetection.POLLING and poll_interval < self.MINIMUM_POLL_INTERVAL: diff --git a/ldclient/testing/file_util.py b/ldclient/testing/file_util.py new file mode 100644 index 00000000..25f5a10a --- /dev/null +++ b/ldclient/testing/file_util.py @@ -0,0 +1,26 @@ +import time + + +def rewrite_in_place(path: str, content: str, pause: float) -> None: + """ + Rewrites a file the way applications and editors often do, without replacing it atomically: + the file is truncated, and the new content is written after the pause. A read during the + pause finds the file empty. A file that does not exist is created with the content. + """ + try: + f = open(path, 'r+') + except FileNotFoundError: + with open(path, 'w') as created: + created.write(content) + return + with f: + f.truncate(0) + f.flush() + time.sleep(pause) + f.write(content) + + +def truncate(path: str) -> None: + """Empties a file in place, which is how an in-place rewrite starts.""" + with open(path, 'r+') as f: + f.truncate(0) diff --git a/ldclient/testing/integrations/test_file_override_source.py b/ldclient/testing/integrations/test_file_override_source.py index a9aea905..abfff9b8 100644 --- a/ldclient/testing/integrations/test_file_override_source.py +++ b/ldclient/testing/integrations/test_file_override_source.py @@ -16,6 +16,7 @@ from ldclient.config import Config as SDKConfig from ldclient.datasystem import custom from ldclient.impl.integrations.files import filedata +from ldclient.impl.integrations.overrides import file_override_source from ldclient.impl.integrations.overrides.file_override_source import ( _FileOverrideSource ) @@ -24,6 +25,7 @@ DuplicateKeysHandling, FileOverrideSourceBuilder ) +from ldclient.testing.file_util import rewrite_in_place, truncate from ldclient.testing.mock_components import HangingSynchronizer from ldclient.testing.stub_util import MockEventProcessor @@ -220,6 +222,25 @@ def test_builder_rejects_watching_without_watchdog(monkeypatch): assert 'watchdog' in str(excinfo.value) +@pytest.mark.parametrize('name', ['overrides.yaml', 'overrides.YML']) +def test_builder_rejects_a_yaml_file_without_pyyaml(monkeypatch, name): + # A YAML file can never load without a YAML parser, so it is a configuration error rather + # than a failure that every reload would report. + import ldclient.integrations.overrides as module + monkeypatch.setattr(module, 'have_yaml', False) + with pytest.raises(ValueError) as excinfo: + FileOverrideSourceBuilder(['/overrides.json', '/' + name]).build(SDKConfig('SDK_KEY')) + assert 'pyyaml' in str(excinfo.value) + assert name in str(excinfo.value) + + +def test_builder_accepts_json_files_without_pyyaml(monkeypatch): + import ldclient.integrations.overrides as module + monkeypatch.setattr(module, 'have_yaml', False) + source = FileOverrideSourceBuilder(['/overrides.json', '/overrides']).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + + # --------------------------------------------------------------------------- # Loading # --------------------------------------------------------------------------- @@ -336,6 +357,48 @@ def test_source_logs_a_present_file_with_no_entries(tmp_path, sources, caplog): require_info_line(caplog, "Flag overrides: none in effect (%s: no entries)" % path) +def wait_for_log(caplog, text: str) -> None: + deadline = time.time() + TEST_TIMEOUT + while not any(text in r.getMessage() for r in caplog.records): + assert time.time() < deadline, "timed out waiting for a log line containing %r" % text + time.sleep(0.01) + + +def test_source_keeps_the_overrides_when_a_json_file_is_emptied(tmp_path, sources, caplog): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + with caplog.at_level(logging.ERROR): + _, sink = build_source(sources, [path], lambda b: b.poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + assert flag_values(sink.require_snapshot()) == {'flag1': True} + + # An empty JSON file is not a document. The overrides stay in effect and, once the file + # has been read again, the failure is logged. + truncate(path) + wait_for_log(caplog, 'no content') + sink.require_no_snapshot() + + # A document with no entries removes the overrides. + write_file(path, '{}') + assert sink.require_snapshot() == ({}, {}) + + +@yaml_required +@watchdog_required +def test_source_removes_the_overrides_of_an_emptied_yaml_file_after_the_retry_delay(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.yaml') + write_file(path, 'flagValues:\n flag1: true\n') + _, sink = build_source(sources, [path], lambda b: b.change_detection(ChangeDetection.WATCHING)) + assert flag_values(sink.require_snapshot()) == {'flag1': True} + + # An empty YAML file is a document with no entries. The watch notices the truncation at + # once, and the file is read again after the retry delay before it is applied, in case it + # is in the middle of an in-place rewrite. + emptied = time.monotonic() + truncate(path) + assert sink.require_snapshot() == ({}, {}) + assert time.monotonic() - emptied >= filedata.DEFAULT_RETRY_DELAY + + # --------------------------------------------------------------------------- # Change detection # --------------------------------------------------------------------------- @@ -437,6 +500,117 @@ def load_then_edit(reloader: filedata.Reloader) -> None: assert flag_values(sink.require_snapshot()) == {'flag1': 'edited-after-the-first-read'} +# --------------------------------------------------------------------------- +# In-place rewrites +# --------------------------------------------------------------------------- + +IN_PLACE_REWRITE_CASES = [ + pytest.param('json', ChangeDetection.POLLING, id='json-polling'), + pytest.param('yaml', ChangeDetection.POLLING, marks=yaml_required, id='yaml-polling'), + pytest.param('json', ChangeDetection.WATCHING, marks=watchdog_required, id='json-watching'), + pytest.param('yaml', ChangeDetection.WATCHING, marks=[watchdog_required, yaml_required], id='yaml-watching'), +] + + +def override_document(file_format: str, value: str) -> str: + if file_format == 'json': + return '{"flagValues": {"flag1": "%s"}}' % value + return 'flagValues:\n flag1: "%s"\n' % value + + +def configure_rewrite_timing(monkeypatch, mode: ChangeDetection) -> float: + """ + Returns the pause of a slow in-place rewrite and sets a retry delay longer than it, so that + the rewrite completes before the retry, even on a slow machine. A watch notices the + truncation at once. A poll notices it at the next examination, so in polling mode the pause + is longer than the poll interval, which makes a poll find the file empty. + """ + if mode == ChangeDetection.WATCHING: + pause, retry_delay = 0.3, 1.5 + else: + pause, retry_delay = FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL + 0.6, 3.0 + monkeypatch.setattr(file_override_source, 'DEFAULT_RETRY_DELAY', retry_delay) + return pause + + +@pytest.mark.parametrize('file_format, mode', IN_PLACE_REWRITE_CASES) +def test_source_never_supplies_a_snapshot_without_the_override_during_an_in_place_rewrite(tmp_path, sources, monkeypatch, file_format, mode): + pause = configure_rewrite_timing(monkeypatch, mode) + path = os.path.join(str(tmp_path), 'overrides.' + file_format) + write_file(path, override_document(file_format, 'a')) + _, sink = build_source(sources, [path], lambda b: b.change_detection(mode).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + assert flag_values(sink.require_snapshot()) == {'flag1': 'a'} + + # The file is empty for the whole pause, and a load finds it empty. The next snapshot holds + # the rewritten override, and the retry that the empty file armed supplies nothing more. + rewrite_in_place(path, override_document(file_format, 'b'), pause) + assert flag_values(sink.require_snapshot()) == {'flag1': 'b'} + sink.require_no_snapshot(file_override_source.DEFAULT_RETRY_DELAY) + + +@pytest.mark.parametrize('file_format, mode', IN_PLACE_REWRITE_CASES) +def test_source_never_supplies_a_snapshot_without_the_override_during_successive_in_place_rewrites(tmp_path, sources, file_format, mode): + # The source's own timing: the default retry delay and a one-second poll interval. An + # operator rewrites the file several times in a row, in place. A watch sees each write and + # the next truncation together; a poll can see a truncation without the content written + # before it. No snapshot may lack the override. + path = os.path.join(str(tmp_path), 'overrides.' + file_format) + write_file(path, override_document(file_format, 'a')) + _, sink = build_source(sources, [path], lambda b: b.change_detection(mode).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + assert flag_values(sink.require_snapshot()) == {'flag1': 'a'} + + values = ['b', 'c', 'd', 'e'] + for value in values: + rewrite_in_place(path, override_document(file_format, value), 0.3) + if mode == ChangeDetection.POLLING: + # The rewrites start about a second apart. + time.sleep(0.7) + + supplied: List[Dict[str, Any]] = [] + while len(supplied) == 0 or supplied[-1] != {'flag1': values[-1]}: + supplied.append(flag_values(sink.require_snapshot())) + assert all(list(snapshot.keys()) == ['flag1'] for snapshot in supplied), supplied + sink.require_no_snapshot(filedata.DEFAULT_RETRY_DELAY + 0.5) + + +@pytest.mark.parametrize('file_format, mode', IN_PLACE_REWRITE_CASES) +def test_client_serves_the_override_throughout_an_in_place_rewrite(tmp_path, monkeypatch, file_format, mode): + # An operator changes an override by rewriting its file in place. A client that evaluates the + # flag throughout serves the old value and then the new one, never the default. + pause = configure_rewrite_timing(monkeypatch, mode) + path = os.path.join(str(tmp_path), 'overrides.' + file_format) + write_file(path, override_document(file_format, 'a')) + source = FileOverrideSourceBuilder([path]).change_detection(mode).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL) + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(source).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + user = Context.create('user-key') + + with LDClient(config, start_wait=0) as client: + served: List[Any] = [] + stop = threading.Event() + + def sample() -> None: + while not stop.is_set(): + value = client.variation('flag1', user, 'default') + if len(served) == 0 or served[-1] != value: + served.append(value) + time.sleep(0.001) + + sampler = threading.Thread(target=sample, daemon=True) + sampler.start() + try: + rewrite_in_place(path, override_document(file_format, 'b'), pause) + deadline = time.time() + TEST_TIMEOUT + while served[-1] != 'b': + assert time.time() < deadline, "the rewritten override was not served; served %r" % served + time.sleep(0.01) + time.sleep(0.5) + finally: + stop.set() + sampler.join(TEST_TIMEOUT) + assert served == ['a', 'b'] + + def reloader_threads() -> Set[threading.Thread]: return {t for t in threading.enumerate() if t.name == 'ldclient.filedata.reloader'} diff --git a/ldclient/testing/integrations/test_filedata.py b/ldclient/testing/integrations/test_filedata.py index 7f9875d5..d6bbc9a8 100644 --- a/ldclient/testing/integrations/test_filedata.py +++ b/ldclient/testing/integrations/test_filedata.py @@ -3,7 +3,7 @@ import threading import time from queue import Empty, Queue -from typing import Any, Dict, List, Set, Tuple +from typing import Any, Dict, List, Optional, Set, Tuple from unittest import mock import pytest @@ -27,6 +27,7 @@ abs_file_paths, have_watchdog, have_yaml, + is_yaml_path, load_files, make_flag_with_value, merge, @@ -34,6 +35,7 @@ read_file ) from ldclient.impl.model import FeatureFlag, Segment +from ldclient.testing.file_util import rewrite_in_place, truncate TEST_TIMEOUT = 5.0 @@ -42,6 +44,9 @@ SHORT_DELAY = 0.05 QUIET_PERIOD = 0.3 +# The pause of the fixture's in-place rewrites, between truncating the file and writing it. +REWRITE_PAUSE = 0.03 + def write_file(path: str, content: str) -> None: with open(path, 'w') as f: @@ -78,30 +83,67 @@ def test_parse_json_document(): assert document.segments['seg1'].included == {'user1'} -def test_parse_chooses_the_parser_the_way_the_file_data_source_does(): - # The parser depends only on whether pyyaml is installed, never on the content. A tab inside - # a string is invalid JSON but valid YAML, so with pyyaml this document parses, and a parser - # chosen from the leading brace would reject it. - if not have_yaml: - pytest.skip("pyyaml is not installed") - document = parse_document(b' \n {"flagValues": {"flag1": "a\tb"}}') - assert document.flag_values == {'flag1': 'a\tb'} +@pytest.mark.parametrize('path', ['/data.yaml', '/data.yml', '/data.YAML', '/data.Yml', 'relative/data.yaml']) +def test_a_yaml_extension_selects_yaml(path): + assert is_yaml_path(path) + + +@pytest.mark.parametrize('path', ['/data.json', '/data', '/data.txt', '/data.yaml.json', '/yaml', '/data.yaml/file', '/data.yamlx']) +def test_any_other_file_is_json(path): + assert not is_yaml_path(path) + + +def test_parse_chooses_the_parser_by_the_extension_not_the_content(): + # A tab inside a string is valid YAML but not valid JSON. Installing pyyaml does not make a + # JSON file YAML: the extension decides, whatever the content looks like. + content = b' \n {"flagValues": {"flag1": "a\tb"}}' + with pytest.raises(ValueError): + parse_document(content) + with pytest.raises(ValueError): + parse_document(b'flagValues:\n flag1: true\n') + if have_yaml: + assert parse_document(content, is_yaml=True).flag_values == {'flag1': 'a\tb'} +@pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed") def test_parse_yaml_document(): - if not have_yaml: - pytest.skip("pyyaml is not installed") - document = parse_document(b'---\nflags:\n flag1:\n key: flag1\n "on": true\nflagValues:\n flag2: value2\nsegments:\n seg1:\n key: seg1\n') + document = parse_document(b'---\nflags:\n flag1:\n key: flag1\n "on": true\nflagValues:\n flag2: value2\nsegments:\n seg1:\n key: seg1\n', is_yaml=True) assert document.flags['flag1'].on is True assert document.flag_values == {'flag2': 'value2'} assert list(document.segments.keys()) == ['seg1'] -def test_parse_empty_document_has_no_entries(): - document = parse_document(b'') - assert document == Document() - document = parse_document(b'{}') - assert document == Document() +def test_parse_yaml_without_pyyaml_fails(monkeypatch): + monkeypatch.setattr(filedata, 'have_yaml', False) + with pytest.raises(ValueError) as excinfo: + parse_document(b'flagValues:\n flag1: true\n', is_yaml=True) + assert 'pyyaml' in str(excinfo.value) + + +@pytest.mark.parametrize('content', [b'', b' \n\t\r\n', b'\xef\xbb\xbf', b'\xef\xbb\xbf \n'], ids=['empty', 'white-space', 'byte-order-mark', 'byte-order-mark-and-white-space']) +def test_parse_empty_json_is_not_a_document(content): + # A file that is rewritten in place is empty for a moment. As JSON it is invalid, so the load + # fails and the last good data stays. + with pytest.raises(ValueError) as excinfo: + parse_document(content) + assert str(excinfo.value) == 'the file has no content, which is not a JSON document' + + +@pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed") +@pytest.mark.parametrize('content', [b'', b' \n\n', b'---\n', b'# no entries yet\n', b'\xef\xbb\xbf'], ids=['empty', 'white-space', 'document-start', 'comment', 'byte-order-mark']) +def test_parse_empty_yaml_has_no_entries(content): + assert parse_document(content, is_yaml=True) == Document() + + +def test_parse_json_object_with_no_members_has_no_entries(): + assert parse_document(b'{}') == Document() + assert parse_document(b' {} \n') == Document() + + +def test_parse_ignores_a_byte_order_mark(): + assert parse_document(b'\xef\xbb\xbf{"flagValues": {"flag1": true}}').flag_values == {'flag1': True} + if have_yaml: + assert parse_document(b'\xef\xbb\xbfflagValues:\n flag1: true\n', is_yaml=True).flag_values == {'flag1': True} def test_parse_fills_in_missing_key_and_version(): @@ -115,7 +157,9 @@ def test_parse_fills_in_missing_key_and_version(): def test_parse_rejects_documents_that_are_not_objects(): if have_yaml: with pytest.raises(ValueError): - parse_document(b'- a\n- b\n') + parse_document(b'- a\n- b\n', is_yaml=True) + with pytest.raises(ValueError): + parse_document(b'["a", "b"]') with pytest.raises(ValueError): parse_document(b'{"flags": ["not", "an", "object"]}') with pytest.raises(ValueError): @@ -127,12 +171,16 @@ def test_parse_rejects_documents_that_are_not_objects(): def test_parse_rejects_malformed_json(): - # With pyyaml the YAML parser reports the error, as in the file data sources. - expected = (ValueError, yaml.YAMLError) if have_yaml else (ValueError,) - with pytest.raises(expected): + with pytest.raises(ValueError): parse_document(b'{"flagValues"') +@pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed") +def test_parse_rejects_malformed_yaml(): + with pytest.raises(yaml.YAMLError): + parse_document(b'flagValues: [', is_yaml=True) + + def test_parse_validates_definition_property_types(): with pytest.raises(ValueError): parse_document(b'{"flags": {"flag1": {"key": "flag1", "version": "not a number"}}}') @@ -156,6 +204,20 @@ def test_read_file_json_and_errors(tmp_path): assert 'error parsing file' in str(excinfo.value) assert path in str(excinfo.value) + write_file(path, '') + with pytest.raises(FileReadError) as excinfo: + read_file(path) + assert 'no content' in str(excinfo.value) + + +@pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed") +def test_read_file_yaml(tmp_path): + path = os.path.join(str(tmp_path), 'data.yaml') + write_file(path, 'flagValues:\n flag1: true\n') + assert read_file(path).flag_values == {'flag1': True} + write_file(path, '') + assert read_file(path) == Document() + def test_abs_file_paths(): paths = abs_file_paths(['relative/data.json', '/absolute/data.json']) @@ -269,6 +331,39 @@ def test_load_files_reports_parse_error_with_path(tmp_path): assert excinfo.value.path == path +@pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed") +def test_load_files_parses_each_file_in_the_format_of_its_extension(tmp_path): + json_path = os.path.join(str(tmp_path), 'first.json') + yaml_path = os.path.join(str(tmp_path), 'second.YML') + write_file(json_path, '{"flagValues": {"flag1": "json"}}') + write_file(yaml_path, 'flagValues:\n flag2: yaml\n') + result = load_files([json_path, yaml_path], DuplicateKeysHandling.FAIL) + assert {key: flag.variations for key, flag in result.flags.items()} == {'flag1': ['json'], 'flag2': ['yaml']} + + # The same YAML content in a file without a YAML extension is not JSON. + write_file(json_path, 'flagValues:\n flag1: json\n') + with pytest.raises(FileReadError) as excinfo: + load_files([json_path, yaml_path], DuplicateKeysHandling.FAIL) + assert excinfo.value.path == json_path + + +@pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed") +def test_load_files_reads_an_empty_yaml_file_as_no_entries_and_fails_on_an_empty_json_file(tmp_path): + json_path = os.path.join(str(tmp_path), 'data.json') + yaml_path = os.path.join(str(tmp_path), 'data.yaml') + write_file(json_path, '{"flagValues": {"flag1": true}}') + write_file(yaml_path, '') + result = load_files([json_path, yaml_path], DuplicateKeysHandling.FAIL) + assert list(result.flags.keys()) == ['flag1'] + assert result.files[1] == FileSummary(path=yaml_path, present=True) + + write_file(json_path, '') + with pytest.raises(FileReadError) as excinfo: + load_files([json_path, yaml_path], DuplicateKeysHandling.FAIL) + assert excinfo.value.path == json_path + assert 'no content' in str(excinfo.value) + + def test_load_files_reports_duplicate_keys(tmp_path): first = os.path.join(str(tmp_path), 'first.json') second = os.path.join(str(tmp_path), 'second.json') @@ -287,8 +382,8 @@ def test_load_files_reports_duplicate_keys(tmp_path): # --------------------------------------------------------------------------- class ReloaderFixture: - def __init__(self, tmp_path, initial_content: str, **kwargs): - self.path = os.path.join(str(tmp_path), 'data.json') + def __init__(self, tmp_path, initial_content: str, name: str = 'data.json', **kwargs): + self.path = os.path.join(str(tmp_path), name) self.applied: Queue = Queue() self.errored: Queue = Queue() write_file(self.path, initial_content) @@ -302,13 +397,16 @@ def __init__(self, tmp_path, initial_content: str, **kwargs): self.reloader = Reloader(**options) def write(self, content: str) -> None: - write_file(self.path, content) + # Applications rewrite data files in place, which is not atomic. The fixture does too, + # deliberately slowly: the file is truncated and written a little later, so a load can + # find it empty, as it would with an application's writes, instead of only by chance. + rewrite_in_place(self.path, content, REWRITE_PAUSE) - def require_applied(self) -> MergeResult: - return take(self.applied) + def require_applied(self, timeout: float = TEST_TIMEOUT) -> MergeResult: + return take(self.applied, timeout) - def require_errored(self) -> Exception: - return take(self.errored) + def require_errored(self, timeout: float = TEST_TIMEOUT) -> Exception: + return take(self.errored, timeout) def require_quiet(self, duration: float = QUIET_PERIOD) -> None: deadline = time.time() + duration @@ -450,16 +548,17 @@ def test_reloader_reports_identical_failure_only_once(make_fixture): # The automatic retries keep failing in the same way. They do not report again. f.require_quiet() - # A different failure is reported. The write is ordinary, not atomic, and a retry that runs - # while the file is being rewritten reads it empty or half written. An empty document is - # valid, so such a retry succeeds and ends the retries; a half-written file is a failure of - # its own. The final content is therefore loaded on demand, and everything that was reported - # is inspected as a whole: the new failure is among the reports, and once it is the current - # failure, the retries that repeat it stay silent. + # A different failure is reported. The write is in place, so a retry can run while the file + # is empty. That retry reads the file again instead of reporting it, and a retry that finds + # it still empty reports that it has no content. The final content is loaded on demand, and + # every report is checked: each one is the new failure or the empty file, the new failure is + # among them, and once it is the current failure, the retries that repeat it stay silent. f.write('{"flags": {"flag1": "not an object"}}') f.reloader.reload_now() - _, errored = f.drain() + applied, errored = f.drain() + assert applied == [] assert any('must be an object' in str(e) for e in errored), errored + assert all('must be an object' in str(e) or 'no content' in str(e) for e in errored), errored f.require_quiet() @@ -695,6 +794,655 @@ def on_error(err: Exception) -> None: f.require_applied() +# --------------------------------------------------------------------------- +# Reloader: files that are found empty +# --------------------------------------------------------------------------- + +yaml_required = pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed") + +# The retry delay of the tests of a file that is found empty. The retry comes long enough after a +# trigger for a test to finish rewriting the file before it, even on a slow machine. +EMPTY_FILE_RETRY_DELAY = 1.5 + +# Shorter than EMPTY_FILE_RETRY_DELAY, so a load that happens within it after a trigger is not +# the retry. +BEFORE_THE_RETRY = 1.0 + +DEFERRAL_LOG = 'waiting for it to be written' + +YAML_FLAG1 = 'flagValues:\n flag1: true\n' +YAML_FLAG2 = 'flagValues:\n flag2: true\n' + + +def flag_keys(result: MergeResult) -> List[str]: + return list(result.flags.keys()) + + +def deferrals_logged(caplog) -> int: + return len([r for r in caplog.records if r.levelno == logging.DEBUG and DEFERRAL_LOG in r.getMessage()]) + + +def wait_for_deferrals(caplog, count: int) -> None: + deadline = time.monotonic() + TEST_TIMEOUT + while deferrals_logged(caplog) < count: + assert time.monotonic() < deadline, "the empty file was not held back" + time.sleep(0.01) + + +def change_modification_time(path: str) -> None: + """ + Makes sure that the file's modification time differs from what it was, as a rewrite changes + it, even on a file system whose timestamps are too coarse to tell two quick writes apart. + """ + modified = os.stat(path).st_mtime_ns + 1000000 + os.utime(path, ns=(modified, modified)) + + +class FakeClock: + """A monotonic clock that only moves when a test advances it.""" + + def __init__(self): + self.now = 1000.0 + + def __call__(self) -> float: + return self.now + + def advance(self, seconds: float) -> None: + self.now += seconds + + +# The retry delay of the tests that time a hold on a fake clock. The timers use the real clock, +# so no retry fires during such a test, and each load is started with reload_now. +HOUR = 3600.0 + + +def make_held_fixture(make_fixture, content: str, name: str = 'data.yaml', **kwargs) -> Tuple[ReloaderFixture, FakeClock]: + clock = FakeClock() + f = make_fixture(content, name=name, retry_delay=HOUR, clock=clock, **kwargs) + f.reloader.reload_now() + f.require_applied() + return f, clock + + +def make_two_file_fixture(make_fixture, tmp_path, first: str, second: str, **kwargs) -> Tuple[ReloaderFixture, FakeClock, str]: + clock = FakeClock() + other = os.path.join(str(tmp_path), 'other.yaml') + write_file(other, second) + f = make_fixture(first, name='data.yaml') + f.reloader = Reloader([f.path, other], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock, **kwargs) + f.reloader.reload_now() + f.require_applied() + return f, clock, other + + +def load(f: ReloaderFixture) -> Optional[MergeResult]: + """Loads the files at once, and returns what was applied, or None when nothing was.""" + f.reloader.reload_now() + try: + return f.applied.get_nowait() + except Empty: + return None + + +def require_held_back(f: ReloaderFixture) -> None: + assert load(f) is None, "the load applied the file instead of holding it back" + assert f.errored.empty(), "the load reported %r" % f.errored.get_nowait() + + +@yaml_required +def test_reloader_holds_back_a_file_until_it_has_stayed_empty_for_exactly_the_retry_delay(make_fixture, caplog): + # A writer that rewrites the file in place truncates it and then writes the new content. A + # load that finds the file empty while its content is in effect keeps the data. In YAML an + # empty file is valid, so applying it would briefly remove every entry. + caplog.set_level(logging.DEBUG) + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + assert deferrals_logged(caplog) == 1 + + clock.advance(0.95 * HOUR) + require_held_back(f) + + clock.advance(0.05 * HOUR) + result = load(f) + assert result is not None and result.flags == {} + + +@yaml_required +def test_reloader_restarts_the_delay_when_the_held_back_file_is_written(make_fixture): + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + + # Rewritten and truncated again between two loads: its state changed, so it is a new rewrite. + clock.advance(0.6 * HOUR) + write_file(f.path, YAML_FLAG2) + truncate(f.path) + change_modification_time(f.path) + require_held_back(f) + + clock.advance(0.6 * HOUR) + require_held_back(f) + + clock.advance(0.4 * HOUR) + result = load(f) + assert result is not None and result.flags == {} + + +@yaml_required +def test_reloader_observes_a_held_back_file_through_a_symbolic_link(make_fixture, tmp_path): + # The configured path is a link to the file that is rewritten, as with a mounted ConfigMap. + # Writing the file changes its state as seen through the link, so the delay starts over. + target = os.path.join(str(tmp_path), 'target.yaml') + link = os.path.join(str(tmp_path), 'link.yaml') + write_file(target, YAML_FLAG1) + try: + os.symlink(target, link) + except (OSError, NotImplementedError): + pytest.skip("symbolic links are not available") + clock = FakeClock() + f = make_fixture(YAML_FLAG1, name='unused.yaml') + f.reloader = Reloader([link], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + truncate(target) + require_held_back(f) + clock.advance(0.6 * HOUR) + write_file(target, YAML_FLAG2) + truncate(target) + change_modification_time(target) + require_held_back(f) + clock.advance(0.6 * HOUR) + require_held_back(f) + + +@yaml_required +def test_reloader_forgets_an_emptying_once_the_file_is_read_with_content(make_fixture): + # A file that was held back, then applied with content, and emptied again much later starts a + # new delay, even if it ends up with the same modification time as at the first emptying, as + # when a tool restores the times of the files it writes. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + first = os.stat(f.path) + require_held_back(f) + write_file(f.path, YAML_FLAG2) + result = load(f) + assert result is not None and flag_keys(result) == ['flag2'] + + clock.advance(10 * HOUR) + truncate(f.path) + os.utime(f.path, ns=(first.st_atime_ns, first.st_mtime_ns)) + require_held_back(f) + + +@yaml_required +def test_reloader_applies_the_written_content_of_a_held_back_file_at_once(make_fixture): + # Two quick rewrites in place: the second truncation follows the first write before any load + # reads the new content, as when both fall in one debounce window. The file is empty at every + # load, and its emptiness is not applied; the content that is finally written is. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + write_file(f.path, YAML_FLAG2) + truncate(f.path) + change_modification_time(f.path) + clock.advance(0.2 * HOUR) + require_held_back(f) + + write_file(f.path, 'flagValues:\n flag3: true\n') + result = load(f) + assert result is not None and flag_keys(result) == ['flag3'] + + +@yaml_required +@pytest.mark.parametrize('content', [' \n\n', '# no entries yet\n', '---\n', 'null\n'], ids=['white-space', 'comment', 'document-marker', 'null']) +def test_reloader_holds_back_a_yaml_file_that_has_no_entries(make_fixture, content): + # A read can see only the start of a YAML file that is being written: white space, a comment + # or a document marker, which the parser reads as nothing. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + write_file(f.path, content) + require_held_back(f) + write_file(f.path, YAML_FLAG2) + result = load(f) + assert result is not None and flag_keys(result) == ['flag2'] + + +@yaml_required +def test_reloader_applies_a_yaml_object_with_no_members_at_once(make_fixture, caplog): + # {} is a document with no entries, written on purpose to remove them, not part of a file. + caplog.set_level(logging.DEBUG) + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + write_file(f.path, '{}\n') + result = load(f) + assert result is not None and result.flags == {} + assert deferrals_logged(caplog) == 0 + + +@yaml_required +def test_reloader_holds_back_files_emptied_together_for_one_retry_delay(make_fixture, tmp_path, caplog): + # Every file is read and classified before any is held back, so two files that are emptied + # together start their delays in the same load and are applied together. + caplog.set_level(logging.DEBUG) + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + truncate(f.path) + truncate(other) + require_held_back(f) + assert deferrals_logged(caplog) == 2 + + clock.advance(HOUR) + result = load(f) + assert result is not None and result.flags == {} + + +@yaml_required +def test_reloader_holds_back_an_emptied_file_while_another_file_changes(make_fixture, tmp_path): + # A change to one file starts a load while another is in the middle of a rewrite. The load + # applies all files or none, so the change waits too. + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + truncate(f.path) + require_held_back(f) + + write_file(other, 'flagValues:\n flag2: false\n') + clock.advance(0.3 * HOUR) + require_held_back(f) + + write_file(f.path, YAML_FLAG1) + result = load(f) + assert result is not None and flag_keys(result) == ['flag1', 'flag2'] + assert result.flags['flag2'].variations == [False] + + +def test_reloader_fails_on_a_file_that_cannot_be_parsed_while_another_is_held_back(make_fixture, tmp_path): + clock = FakeClock() + other = os.path.join(str(tmp_path), 'other.json') + write_file(other, '{"flagValues": {"flag2": true}}') + f = make_fixture('{"flagValues": {"flag1": true}}') + f.reloader = Reloader([f.path, other], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + # One file is mid-rewrite and the other is broken. The broken file is reported at once. + truncate(f.path) + write_file(other, '{"flagValues"') + assert load(f) is None + err = f.errored.get_nowait() + assert isinstance(err, FileReadError) + assert err.path == other + + +@yaml_required +def test_reloader_stops_holding_back_a_file_that_keeps_being_emptied_after_five_retry_delays(make_fixture): + # Every load finds the file empty with a new modification time, so its delay keeps starting + # over. The run of loads that hold it back ends after five retry delays. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + for _ in range(9): + clock.advance(0.5 * HOUR) + change_modification_time(f.path) + require_held_back(f) + + clock.advance(0.5 * HOUR) + change_modification_time(f.path) + result = load(f) + assert result is not None and result.flags == {} + + +@yaml_required +def test_reloader_stops_holding_back_files_rewritten_in_turn_after_five_retry_delays(make_fixture, tmp_path): + # Two files are rewritten in turn, so every load finds one of them empty, and each is read + # with content in between. The run of loads that hold files back still ends after five retry + # delays. + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + paths = [f.path, other] + contents = [YAML_FLAG1, YAML_FLAG2] + truncate(f.path) + require_held_back(f) + for i in range(1, 10): + clock.advance(0.5 * HOUR) + write_file(paths[(i - 1) % 2], contents[(i - 1) % 2]) + truncate(paths[i % 2]) + require_held_back(f) + + clock.advance(0.5 * HOUR) + write_file(paths[1], contents[1]) + truncate(paths[0]) + result = load(f) + assert result is not None and flag_keys(result) == ['flag2'] + + +@yaml_required +def test_reloader_holds_back_a_file_that_is_emptied_again_after_its_content_was_applied(make_fixture): + # A load that holds no file back ends the run, so a later rewrite is held back on its own. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + write_file(f.path, YAML_FLAG2) + clock.advance(0.5 * HOUR) + result = load(f) + assert result is not None and flag_keys(result) == ['flag2'] + + clock.advance(10 * HOUR) + truncate(f.path) + require_held_back(f) + clock.advance(HOUR) + result = load(f) + assert result is not None and result.flags == {} + + +def test_reloader_holds_back_a_json_file_that_was_empty_for_long_when_it_is_rewritten(make_fixture, caplog): + # A JSON file that stays empty fails to load once the delay is over, and keeps failing. The + # failures end the run of holds, so when the file is rewritten in place much later, the new + # truncation is held back like any other, without another report. + caplog.set_level(logging.DEBUG) + f, clock = make_held_fixture(make_fixture, '{"flagValues": {"flag1": true}}', name='data.json') + truncate(f.path) + require_held_back(f) + + clock.advance(HOUR) + assert load(f) is None + assert 'no content' in str(f.errored.get_nowait()) + clock.advance(HOUR) + assert load(f) is None + assert f.errored.empty() + + clock.advance(10 * HOUR) + held_before = deferrals_logged(caplog) + truncate(f.path) + change_modification_time(f.path) + require_held_back(f) + assert deferrals_logged(caplog) == held_before + 1 + + write_file(f.path, '{"flagValues": {"flag1": false}}') + result = load(f) + assert result is not None and result.flags['flag1'].variations == [False] + + +@yaml_required +def test_reloader_does_not_hold_back_a_file_that_is_empty_from_the_start(make_fixture, tmp_path, caplog): + # An empty YAML file next to the data, such as a placeholder for later entries, has never had + # content. It does not hold back the changes of the other file. + caplog.set_level(logging.DEBUG) + clock = FakeClock() + placeholder = os.path.join(str(tmp_path), 'placeholder.yaml') + write_file(placeholder, '') + f = make_fixture('{"flagValues": {"flag1": true}}') + f.reloader = Reloader([f.path, placeholder], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + assert flag_keys(f.require_applied()) == ['flag1'] + + write_file(f.path, '{"flagValues": {"flag1": false}}') + result = load(f) + assert result is not None and result.flags['flag1'].variations == [False] + assert deferrals_logged(caplog) == 0 + + +@yaml_required +def test_reloader_applies_a_file_that_is_deleted_and_then_created_empty_at_once(make_fixture): + f, clock = make_held_fixture(make_fixture, YAML_FLAG1, skip_missing_paths=True) + + # Deleting a file that had content removes its data at once. + os.remove(f.path) + result = load(f) + assert result is not None and result.flags == {} + + # A missing file contributed no content, so the empty file that replaces it is applied at once. + write_file(f.path, '') + result = load(f) + assert result is not None and result.flags == {} + assert result.files == [FileSummary(path=f.path, present=True)] + + +@yaml_required +def test_reloader_does_not_hold_back_a_file_whose_content_moved_away(make_fixture, tmp_path): + # The content moves from one file to the other. The digest of the files' bytes does not say + # which file holds them, so it is the same as before and the load is skipped as unchanged. + # The file that no longer exists contributes no content, so when it is created empty, that + # is applied at once. + clock = FakeClock() + other = os.path.join(str(tmp_path), 'other.yaml') + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([f.path, other], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, skip_missing_paths=True, retry_delay=HOUR, skip_unchanged=True, clock=clock) + f.reloader.reload_now() + assert flag_keys(f.require_applied()) == ['flag1'] + + write_file(other, YAML_FLAG1) + os.remove(f.path) + assert load(f) is None + + write_file(f.path, '') + result = load(f) + assert result is not None and flag_keys(result) == ['flag1'] + assert result.files == [FileSummary(path=f.path, present=True), FileSummary(path=other, present=True, flags=1)] + + +@yaml_required +def test_reloader_holds_back_a_file_whose_content_was_moved_to_it_from_another(make_fixture, tmp_path): + # The file is renamed to the other configured path (mv data.yaml other.yaml), so the bytes are + # the same as before and the load is skipped as unchanged. The other path's content is in + # effect now, so an in-place rewrite of it is held back. + clock = FakeClock() + other = os.path.join(str(tmp_path), 'other.yaml') + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([f.path, other], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, skip_missing_paths=True, retry_delay=HOUR, skip_unchanged=True, clock=clock) + f.reloader.reload_now() + assert flag_keys(f.require_applied()) == ['flag1'] + + os.replace(f.path, other) + assert load(f) is None + + truncate(other) + require_held_back(f) + write_file(other, YAML_FLAG2) + result = load(f) + assert result is not None and flag_keys(result) == ['flag2'] + + +@yaml_required +def test_reloader_applies_an_emptied_file_at_once_when_retries_are_off(make_fixture, caplog): + # Nothing would load the file later, so the empty file is parsed as it is. + caplog.set_level(logging.DEBUG) + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + result = load(f) + assert result is not None and result.flags == {} + assert deferrals_logged(caplog) == 0 + + +def test_reloader_reports_an_emptied_json_file_at_once_when_retries_are_off(make_fixture): + f = make_fixture('{"flagValues": {"flag1": true}}') + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + assert load(f) is None + assert 'no content' in str(f.errored.get_nowait()) + + +def test_reloader_hold_is_not_a_failure(make_fixture, caplog): + # A rewrite in place that writes the same content again changes nothing. The load that finds + # the file empty reports nothing, and is not a failure that a later load recovers from, so + # the unchanged content is not applied again. + caplog.set_level(logging.DEBUG) + content = '{"flagValues": {"flag1": true}}' + f, clock = make_held_fixture(make_fixture, content, name='data.json', skip_unchanged=True) + truncate(f.path) + require_held_back(f) + write_file(f.path, content) + assert load(f) is None + assert f.errored.empty() + assert not any(r.levelno >= logging.WARNING for r in caplog.records), [r.getMessage() for r in caplog.records] + + +@yaml_required +def test_reloader_initial_load_reads_an_empty_yaml_file_as_no_entries(make_fixture): + f = make_fixture('', name='data.yaml', retry_delay=EMPTY_FILE_RETRY_DELAY) + f.reloader.reload_now() + result = f.require_applied() + assert result.flags == {} + assert result.files == [FileSummary(path=f.path, present=True)] + + +def test_reloader_initial_load_fails_on_an_empty_json_file(make_fixture): + f = make_fixture('', retry_delay=EMPTY_FILE_RETRY_DELAY) + f.reloader.reload_now() + assert 'no content' in str(f.require_errored()) + f.require_quiet(0.1) + + +# Real-clock tests: the retries that holds arm do fire. + +@yaml_required +def test_reloader_holds_back_an_emptied_file_until_its_write_triggers_a_load(make_fixture, caplog): + caplog.set_level(logging.DEBUG) + f = make_fixture(YAML_FLAG1, name='data.yaml', retry_delay=EMPTY_FILE_RETRY_DELAY, skip_unchanged=True) + f.reloader.reload_now() + assert flag_keys(f.require_applied()) == ['flag1'] + + truncate(f.path) + f.reloader.trigger() + wait_for_deferrals(caplog, 1) + f.require_quiet(0.1) + + # The write triggers a load that applies the new content. Nothing was applied in between. + write_file(f.path, YAML_FLAG2) + f.reloader.trigger() + assert flag_keys(f.require_applied(BEFORE_THE_RETRY)) == ['flag2'] + f.require_quiet(0.1) + + +@yaml_required +def test_reloader_applies_an_emptied_yaml_file_that_stays_empty_at_the_retry(make_fixture, caplog): + caplog.set_level(logging.DEBUG) + f = make_fixture(YAML_FLAG1, name='data.yaml', retry_delay=EMPTY_FILE_RETRY_DELAY) + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + started = time.monotonic() + f.reloader.trigger() + wait_for_deferrals(caplog, 1) + + # The retry that the hold armed comes when the file has stayed empty and unmodified for the + # retry delay, so that first retry applies it as a document with no entries. + assert f.require_applied().flags == {} + elapsed = time.monotonic() - started + assert EMPTY_FILE_RETRY_DELAY <= elapsed < 1.5 * EMPTY_FILE_RETRY_DELAY + assert deferrals_logged(caplog) == 1 + + +@yaml_required +def test_reloader_a_further_hold_does_not_put_off_the_retry(make_fixture, caplog): + # A load that holds the file back again arms the retry for the time that is left of the + # delay, so the file is still applied one retry delay after it was found empty. + caplog.set_level(logging.DEBUG) + retry_delay = 3.0 + f = make_fixture(YAML_FLAG1, name='data.yaml', retry_delay=retry_delay) + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + started = time.monotonic() + f.reloader.trigger() + wait_for_deferrals(caplog, 1) + time.sleep(1.0) + f.reloader.trigger() + wait_for_deferrals(caplog, 2) + + assert f.require_applied().flags == {} + elapsed = time.monotonic() - started + assert retry_delay <= elapsed < retry_delay + 0.65 + + +def test_reloader_reports_an_emptied_json_file_only_at_the_retry_and_keeps_the_data(make_fixture, caplog): + caplog.set_level(logging.DEBUG) + f = make_fixture('{"flagValues": {"flag1": true}}', retry_delay=EMPTY_FILE_RETRY_DELAY) + f.reloader.reload_now() + f.require_applied() + + # The load that the trigger starts holds the file back instead of reporting it. The retry + # finds it still empty and unmodified, and an empty file is not a JSON document. + truncate(f.path) + f.reloader.trigger() + wait_for_deferrals(caplog, 1) + f.require_quiet(0.1) + err = f.require_errored() + assert isinstance(err, FileReadError) + assert 'no content' in str(err) + f.require_quiet(0.1) + + +@yaml_required +def test_reloader_holds_back_a_second_rewrite_that_the_retry_of_the_first_finds_empty(make_fixture, caplog): + # The first rewrite is held back. Its content is written and the file is truncated again by + # a second rewrite before any load reads that content, as when a polled file is rewritten + # about a second apart and the poll sees only the second truncation. The retry that the + # first rewrite armed finds the file empty again, and must not apply it. + caplog.set_level(logging.DEBUG) + f = make_fixture(YAML_FLAG1, name='data.yaml', retry_delay=EMPTY_FILE_RETRY_DELAY) + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + f.reloader.trigger() + wait_for_deferrals(caplog, 1) + + # No trigger: only the retry reads the file. + write_file(f.path, YAML_FLAG2) + truncate(f.path) + change_modification_time(f.path) + wait_for_deferrals(caplog, 2) + f.require_quiet(0.2) + + write_file(f.path, 'flagValues:\n flag3: true\n') + f.reloader.trigger() + assert flag_keys(f.require_applied(BEFORE_THE_RETRY)) == ['flag3'] + + +@yaml_required +def test_reloader_retry_that_finds_a_file_emptied_holds_it_back(make_fixture, caplog): + # A retry left over from a failure can run while the file is being rewritten in place. It + # holds the file back too, rather than applying it empty. + caplog.set_level(logging.DEBUG) + f = make_fixture(YAML_FLAG1, name='data.yaml', retry_delay=EMPTY_FILE_RETRY_DELAY) + f.reloader.reload_now() + f.require_applied() + write_file(f.path, 'flagValues: [') + f.reloader.trigger() + f.require_errored() + + # No trigger: only the retry reads the file. + truncate(f.path) + wait_for_deferrals(caplog, 1) + write_file(f.path, YAML_FLAG2) + assert flag_keys(f.require_applied()) == ['flag2'] + f.require_quiet(0.1) + + +@yaml_required +def test_reloader_retry_left_over_from_a_hold_applies_nothing_after_a_load_on_demand(make_fixture, caplog): + caplog.set_level(logging.DEBUG) + f = make_fixture(YAML_FLAG1, name='data.yaml', retry_delay=EMPTY_FILE_RETRY_DELAY, skip_unchanged=True) + f.reloader.reload_now() + f.require_applied() + truncate(f.path) + f.reloader.trigger() + wait_for_deferrals(caplog, 1) + + # A load on demand reads the rewritten file before the retry does. + write_file(f.path, YAML_FLAG2) + f.reloader.reload_now() + assert flag_keys(f.require_applied()) == ['flag2'] + + # The retry still runs. It finds the content that was just applied, so it applies and + # reports nothing. + f.require_quiet(EMPTY_FILE_RETRY_DELAY + 0.5) + + # --------------------------------------------------------------------------- # Poller # --------------------------------------------------------------------------- From 57bb3945ed41590f5f15d960171f133ff735e2d6 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 3 Nov 2026 03:00:07 +0000 Subject: [PATCH 23/26] fix: Bound each held back file on its own and time it from its own read The hold of an emptied file was bounded by the run of loads that held any file back. A failing load ended the run, so a file kept written while empty was held without end while another file failed, and one file reaching the bound applied another file in the middle of an ordinary rewrite. Each file now has its own bound, from when a load first held it back. The bound starts over only at a load that holds nothing back, for every file that load got to, including one that failed to be read or parsed, whatever the load's outcome, and when data is applied or a load is skipped as unchanged. So files emptied in turn stay bounded, a file held during another file's failure is bounded afresh, and the in-place save that fixes a typo is held back like the one that made it. A file read with content, even content that fails to parse, or found missing ends its empty state, so its next emptying starts a new delay. The clock is read after each file's own read, and again for the retry the hold arms, so a slow read of another file does not change a file's delay. The retry after a hold logs that the files are read again because a file was found empty. --- ldclient/impl/integrations/files/filedata.py | 238 +++++---- ldclient/integrations/overrides.py | 2 +- .../testing/integrations/test_filedata.py | 459 ++++++++++++++++++ 3 files changed, 616 insertions(+), 83 deletions(-) diff --git a/ldclient/impl/integrations/files/filedata.py b/ldclient/impl/integrations/files/filedata.py index 41a36ff7..3864f659 100644 --- a/ldclient/impl/integrations/files/filedata.py +++ b/ldclient/impl/integrations/files/filedata.py @@ -311,20 +311,21 @@ def _read_files(paths: List[str], skip_missing_paths: bool) -> List[_FileRead]: Reads the raw content of all of the given files in order. Raises :class:`FileReadError` when a file cannot be read, including a missing file unless ``skip_missing_paths`` is true. """ - contents: List[_FileRead] = [] - for path in paths: - try: - with open(path, "rb") as f: - contents.append(_FileRead(path, f.read())) - except FileNotFoundError: - if skip_missing_paths: - log.debug("File %s does not exist; it contributes no data", path) - contents.append(_FileRead(path, None)) - continue - raise FileReadError(path, "unable to read file: the file does not exist") - except OSError as e: - raise FileReadError(path, "unable to read file: %s" % e) - return contents + return [_read_file(path, skip_missing_paths) for path in paths] + + +def _read_file(path: str, skip_missing_paths: bool) -> _FileRead: + """Reads the raw content of one file, as :func:`_read_files` does.""" + try: + with open(path, "rb") as f: + return _FileRead(path, f.read()) + except FileNotFoundError: + if skip_missing_paths: + log.debug("File %s does not exist; it contributes no data", path) + return _FileRead(path, None) + raise FileReadError(path, "unable to read file: the file does not exist") + except OSError as e: + raise FileReadError(path, "unable to read file: %s" % e) def _parse_file(path: str, raw: bytes) -> Optional[Document]: @@ -391,19 +392,29 @@ def _observe_all(paths: List[str]) -> List[FileState]: return [_observe(path) for path in paths] -# Loads stop holding files back once they have held some back for this many retry delays in a -# row. +# A file is held back for at most this many retry delays after a load first held it back, until +# data is applied. _MAX_HOLD_RETRY_DELAYS = 5 @dataclass class _EmptyFile: - """A file whose content is in effect and that loads have found with no entries since.""" + """A file whose content is in effect and that loads have found with no entries.""" - since: float - """Since when the file has had no entries without being written, on the reloader's clock.""" - state: FileState + since: Optional[float] = None + """ + Since when the file has had no entries without being written, on the reloader's clock, or + None when the last load that got to it read content, even content that failed to parse, or + found it missing. + """ + state: FileState = None """The file's state then: its modification time and size, through any symbolic link.""" + first_held: Optional[float] = None + """ + When a load first held the file back, or None. It bounds how long the file is held back. + It is kept while loads hold files back, so files emptied in turn stay bounded, and cleared by + a load that holds nothing back and gets to the file, or when data is applied. + """ class Reloader: @@ -419,11 +430,13 @@ class Reloader: the current data without reporting a failure, and the other files' changes wait too, since a load applies all files or none. The file is held back until it has stayed so, without being written, for the retry delay; a change of its modification time or size starts the delay - over. Loads stop holding files back once they have held some back for five retry delays in a - row, so files that keep being rewritten are applied in the end. A file whose content is not - in effect, such as a placeholder that has always been empty, is never held back. Every file - is read and parsed before any is held back, so files emptied together wait together, and a - file that cannot be parsed fails the load even while another is held back. + over. Each file is held back for at most five retry delays after a load first held it back, + until a load holds nothing back, so a file that keeps being rewritten is applied in the end + without cutting short another file's rewrite. A file whose content is not in effect, such as a + placeholder that has always been empty, is never held back. Every file is read and parsed + before any is held back, so files emptied together wait together, and a file that cannot be + parsed fails the load even while another is held back. The clock is read after each file's + own read, so a slow read of another file does not shorten a file's delay. The worker thread starts on the first :meth:`reload_now` or :meth:`trigger` call, so a reloader that is constructed but never used does not leak a thread. @@ -480,6 +493,8 @@ def __init__( self._started = False self._debounce_deadline: Optional[float] = None self._retry_deadline: Optional[float] = None + # Whether the pending retry was armed by a load that held files back. + self._retry_after_hold = False # Serializes the load work between reload_now and the worker thread. The fields below are # only touched while holding it. @@ -489,14 +504,15 @@ def __init__( # Whether each configured file contributed content to the data in effect. A missing file, # or one with no entries, contributed none. self._applied_had_content: Dict[str, bool] = {} - # The files whose content is in effect and that loads have found with no entries since. - # A file is removed when a load reads it with entries. + # The files whose content is in effect and that loads have found with no entries. All are + # forgotten when data is applied; failures change nothing. self._empty_files: Dict[str, _EmptyFile] = {} - # When the current run of loads that hold files back began, or None when the last load - # held none back. - self._holding_since: Optional[float] = None - # Set by a load that holds files back. + # Whether the last load held files back, which the log line of the retry it arms names. self._held_back = False + # Whether the current load has held a file back, even if it then fails. + self._holding_this_load = False + # The delay of the last retry that was armed, for tests. + self._last_requested_retry_delay: Optional[float] = None def reload_now(self) -> None: """ @@ -505,13 +521,8 @@ def reload_now(self) -> None: """ self._ensure_started() retry_after = self._reload() - if retry_after is not None and self._retry_delay > 0: - with self._cond: - if not self._closed: - deadline = time.monotonic() + retry_after - if self._retry_deadline is None or deadline < self._retry_deadline: - self._retry_deadline = deadline - self._cond.notify() + if retry_after is not None: + self._arm_retry(retry_after, keep_sooner=True) def trigger(self) -> None: """ @@ -535,6 +546,24 @@ def close(self) -> None: self._closed = True self._cond.notify_all() + def _arm_retry(self, retry_after: float, keep_sooner: bool) -> None: + """ + Arms the automatic retry for the given time from now, unless retries are off. With + ``keep_sooner``, a retry that is already due sooner stays as it is. + """ + if self._retry_delay <= 0: + return + with self._cond: + if self._closed: + return + deadline = time.monotonic() + retry_after + if keep_sooner and self._retry_deadline is not None and self._retry_deadline <= deadline: + return + self._retry_deadline = deadline + self._retry_after_hold = self._held_back + self._last_requested_retry_delay = retry_after + self._cond.notify() + def _ensure_started(self) -> None: with self._cond: if self._closed or self._started: @@ -546,6 +575,7 @@ def _ensure_started(self) -> None: def _run(self) -> None: while True: is_retry = False + after_hold = False with self._cond: while True: if self._closed: @@ -568,10 +598,13 @@ def _run(self) -> None: else: self._retry_deadline = None is_retry = True + after_hold = self._retry_after_hold break - if is_retry: - log.debug("Loading flag data again after an earlier failed or held back load") + if is_retry and after_hold: + log.debug("Reading flag data again after a file was found empty") + elif is_retry: + log.debug("Retrying flag data load after earlier failure") else: log.info("Reloading flag data after detecting a change") try: @@ -579,10 +612,8 @@ def _run(self) -> None: except Exception as e: log.exception("Unexpected error while reloading flag data: %s", e) retry_after = None - if retry_after is not None and self._retry_delay > 0: - with self._cond: - if not self._closed: - self._retry_deadline = time.monotonic() + retry_after + if retry_after is not None: + self._arm_retry(retry_after, keep_sooner=False) def _is_closed(self) -> bool: with self._cond: @@ -598,26 +629,36 @@ def _reload(self) -> Optional[float]: if self._is_closed(): return None self._held_back = False - retry_after = self._load() - if not self._held_back: - # A run of loads that hold files back ends with any load that does not. - self._holding_since = None + self._holding_this_load = False + reached: List[str] = [] + retry_after = self._load(reached) + if not self._holding_this_load: + # A load that holds nothing back, whether it applies, is skipped or fails, starts the + # bound over for every file that it got to, including one that failed to be read or + # parsed: such a file is no longer held back, so its next rewrite is a new one. A + # file that the load did not get to keeps its bound. + for path in reached: + empty = self._empty_files.get(path) + if empty is not None: + empty.first_held = None return retry_after - def _load(self) -> Optional[float]: - try: - contents = _read_files(self._paths, self._skip_missing_paths) - except Exception as e: - return self._fail(e) - - # Every file is parsed and classified before any is held back, so that every file that - # is found with no entries starts its delay in this load. - now = self._clock() + def _load(self, reached: List[str]) -> Optional[float]: + """The body of :meth:`_reload`. It records in ``reached`` each file that it gets to.""" + # Every file is read, parsed and classified before any is held back, so that every file + # that is found with no entries starts its delay in this load. held_back: List[str] = [] parsed: List[Tuple[_FileRead, Optional[Document]]] = [] had_content: Dict[str, bool] = {} - for read in contents: + for path in self._paths: + reached.append(path) + try: + read = _read_file(path, self._skip_missing_paths) + except Exception as e: + return self._fail(e) if read.raw is None: + # A missing file is not an empty one. + self._end_empty(read.path) had_content[read.path] = False parsed.append((read, None)) continue @@ -631,22 +672,28 @@ def _load(self) -> Optional[float]: try: document = _parse_file(read.path, read.raw) except FileReadError as e: + # Content that cannot be parsed is not an empty file either. + self._end_empty(read.path) return self._fail(e) has_document = document is not None if has_document: - self._empty_files.pop(read.path, None) - elif self._should_hold_back(read.path, now): + self._end_empty(read.path) + # The clock is read after this file's own read, so that a slow read of another file + # does not shorten its delay. + elif self._should_hold_back(read.path, self._clock()): held_back.append(read.path) + self._holding_this_load = True continue - elif not has_content(read.raw): + # The file was read and is not held back, even if it then fails to parse. + had_content[read.path] = has_document + if not has_content(read.raw): try: document = _parse_file(read.path, read.raw) except FileReadError as e: return self._fail(e) - had_content[read.path] = has_document parsed.append((read, document)) if len(held_back) > 0: - return self._hold_back(held_back, now) + return self._hold_back(held_back, self._clock()) try: merged, digest = _merge_files(parsed, self._duplicate_keys_handling) @@ -664,8 +711,8 @@ def _load(self) -> Optional[float]: if self._skip_unchanged and not recovering and digest == self._last_good_digest: # The digest does not say which file holds which bytes, so content that moved from # one file to another is skipped as unchanged, and what each file contributes has to - # be remembered here too. - self._applied_had_content.update(had_content) + # be remembered here too. The data in effect is confirmed, so the bounds start over. + self._remember_applied(had_content) return None # Nothing is remembered until the consumer has accepted the result. A result the @@ -677,9 +724,24 @@ def _load(self) -> Optional[float]: return self._fail(e) self._last_error_message = None self._last_good_digest = digest - self._applied_had_content.update(had_content) + self._remember_applied(had_content) return None + def _end_empty(self, path: str) -> None: + """ + Ends the empty state of a file that a load read with content, even content that cannot be + parsed, or found missing, so that it starts a new delay when it is found empty again. Its + bound is kept until a load holds nothing back. + """ + empty = self._empty_files.get(path) + if empty is not None: + empty.since = None + + def _remember_applied(self, had_content: Dict[str, bool]) -> None: + """Records what each file contributed to the data in effect, and forgets the empty files.""" + self._applied_had_content.update(had_content) + self._empty_files.clear() + def _should_hold_back(self, path: str, now: float) -> bool: """ Decides whether a file that a load found with no entries is held back. A file whose @@ -687,41 +749,53 @@ def _should_hold_back(self, path: str, now: float) -> bool: It is held back until it has stayed so, without being written, for the retry delay. A change of its state, its modification time and size read through any symbolic link, means that it was written since a load last found it so, for example rewritten and - truncated again between two loads, so the delay starts over. Loads stop holding files - back once they have held some back for five retry delays in a row, so that a file that - keeps being written while every load finds it empty, or files that are rewritten in turn, - are not held back without end. A file whose content is not in effect is parsed right - away, and so is any file when retries are off, since nothing would read it again. + truncated again between two loads, so the delay starts over. A file that has stayed so + for the whole delay is not held back. Each file is held back for at most five retry + delays after a load first held it back, so that a file that keeps being written while + every load finds it empty, or files that are emptied in turn, are not held back without + end. The bound starts over only at a load that holds nothing back, for every file that + load got to, including one that failed to be read or parsed, whether the load applies, is + skipped or fails. A file after the one that failed keeps its bound. A file whose content is not in effect + is parsed right away, and so is any file when retries are off, since nothing would read it + again. """ if self._retry_delay <= 0 or not self._applied_had_content.get(path, False): return False state = _observe(path) empty = self._empty_files.get(path) if empty is None: - empty = _EmptyFile(since=now, state=state) + empty = _EmptyFile() self._empty_files[path] = empty - elif empty.state != state: + if empty.since is None or empty.state != state: empty.since = now empty.state = state - holding_since = now if self._holding_since is None else self._holding_since - return now - empty.since < self._retry_delay and now - holding_since < _MAX_HOLD_RETRY_DELAYS * self._retry_delay + if now - empty.since >= self._retry_delay: + # The file stayed empty for the whole delay. Its bound starts over only at a load that + # holds nothing back, so that files emptied in turn do not keep each other held back. + return False + if empty.first_held is not None and now - empty.first_held >= _MAX_HOLD_RETRY_DELAYS * self._retry_delay: + return False + if empty.first_held is None: + empty.first_held = now + return True def _hold_back(self, held_back: List[str], now: float) -> Optional[float]: """ Keeps the current data for a load that holds files back. Whatever the files' format, nothing is applied or reported. The returned time arms the retry for when the first of - the files has stayed so for the retry delay, or when the loads have held files back for - as long as they may, plus a tenth of the retry delay, so that a timer that fires a little - early still finds the delay over. The change signal for a write starts a load sooner. + the files has stayed so for the retry delay, or has been held back for as long as it may + be, plus a tenth of the retry delay, so that a timer that fires a little early still + finds the time over. The change signal for a write starts a load sooner. """ if self._is_closed(): return None - if self._holding_since is None: - self._holding_since = now - until = self._holding_since + _MAX_HOLD_RETRY_DELAYS * self._retry_delay + until = float("inf") for path in held_back: log.debug("File %s has no entries, possibly while it is being rewritten; waiting for it to be written before applying it", path) - until = min(until, self._empty_files[path].since + self._retry_delay) + empty = self._empty_files[path] + since = now if empty.since is None else empty.since + first_held = now if empty.first_held is None else empty.first_held + until = min(until, since + self._retry_delay, first_held + _MAX_HOLD_RETRY_DELAYS * self._retry_delay) self._held_back = True return max(until - now, 0.0) + self._retry_delay / 10 diff --git a/ldclient/integrations/overrides.py b/ldclient/integrations/overrides.py index 03ecbee4..7ef71954 100644 --- a/ldclient/integrations/overrides.py +++ b/ldclient/integrations/overrides.py @@ -82,7 +82,7 @@ class FileOverrideSourceBuilder(OverrideSourceBuilder): empty, or with only a YAML comment or document marker, may be in the middle of a rewrite. The source then keeps the overrides in effect, and changes to the other files wait too, since a reload applies all files or none, until the file has stayed so without being written for - about a second, and for at most about five seconds while files keep being rewritten. After + about a second, and for at most about five seconds while it keeps being rewritten. After that, a YAML file is applied with no entries, and an empty JSON file fails to load and keeps the overrides in effect. YAML that is cut off between two entries can still parse, so for a YAML file, replacing its entire contents in one operation, such as renaming a fully written diff --git a/ldclient/testing/integrations/test_filedata.py b/ldclient/testing/integrations/test_filedata.py index d6bbc9a8..1ca6e9db 100644 --- a/ldclient/testing/integrations/test_filedata.py +++ b/ldclient/testing/integrations/test_filedata.py @@ -976,6 +976,69 @@ def test_reloader_forgets_an_emptying_once_the_file_is_read_with_content(make_fi require_held_back(f) +@yaml_required +def test_reloader_holds_back_a_file_emptied_again_after_a_read_with_content_whose_state_looks_the_same(make_fixture, tmp_path): + # On a file system that records modification times coarsely, a file emptied a second time can + # look exactly as it did the first time. The read of its content in between still starts its + # delay over. The other file is kept written while empty, so that no load applies. + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + truncate(f.path) + first_emptying = os.stat(f.path) + truncate(other) + require_held_back(f) + + clock.advance(50 / 60 * HOUR) + write_file(f.path, YAML_FLAG1) + change_modification_time(other) + require_held_back(f) + + clock.advance(20 / 60 * HOUR) + truncate(f.path) + os.utime(f.path, ns=(first_emptying.st_atime_ns, first_emptying.st_mtime_ns)) + change_modification_time(other) + require_held_back(f) + + # The other file gets its content back while this one is still within its new delay. + clock.advance(10 / 60 * HOUR) + write_file(other, YAML_FLAG2) + require_held_back(f) + + +@yaml_required +def test_reloader_holds_back_a_file_rewritten_with_the_same_content_again_later(make_fixture): + # A tool rewrites the file in place with the same content now and then. The load after each + # rewrite is skipped as unchanged, which confirms the data in effect like an application, so + # a later rewrite is held back on its own and not cut short by the first one's bound. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1, skip_unchanged=True) + truncate(f.path) + require_held_back(f) + write_file(f.path, YAML_FLAG1) + assert load(f) is None + + clock.advance(6 * HOUR) + truncate(f.path) + require_held_back(f) + + +@yaml_required +def test_reloader_holds_back_a_file_emptied_again_after_unparsable_content_whose_state_looks_the_same(make_fixture): + # The file is emptied and held back, then saved with content that cannot be parsed, then + # emptied again with the modification time of its first emptying, as a coarse file system can + # make it look. Reading content ended its empty state, so it starts a new delay. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + first_emptying = os.stat(f.path) + require_held_back(f) + write_file(f.path, 'flagValues: [') + assert load(f) is None + assert isinstance(f.errored.get_nowait(), FileReadError) + + clock.advance(70 / 60 * HOUR) + truncate(f.path) + os.utime(f.path, ns=(first_emptying.st_atime_ns, first_emptying.st_mtime_ns)) + require_held_back(f) + + @yaml_required def test_reloader_applies_the_written_content_of_a_held_back_file_at_once(make_fixture): # Two quick rewrites in place: the second truncation follows the first write before any load @@ -1112,6 +1175,385 @@ def test_reloader_stops_holding_back_files_rewritten_in_turn_after_five_retry_de assert result is not None and flag_keys(result) == ['flag2'] +@yaml_required +def test_reloader_measures_a_file_from_its_own_read_when_an_earlier_file_reads_slowly(make_fixture, tmp_path, monkeypatch): + # The read of the earlier file stalls while the later file is being rewritten in place. The + # later file's delay counts from when it was read, not from when the load began. + clock = FakeClock() + slow = os.path.join(str(tmp_path), 'slow.yaml') + write_file(slow, YAML_FLAG2) + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([slow, f.path], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + original = filedata._read_file + stall = [True] + + def read_file(path: str, skip_missing_paths: bool): + read = original(path, skip_missing_paths) + if path == slow and stall[0]: + truncate(f.path) + clock.advance(0.9 * HOUR) + return read + + monkeypatch.setattr(filedata, '_read_file', read_file) + require_held_back(f) + stall[0] = False + clock.advance(0.2 * HOUR) + require_held_back(f) + + write_file(f.path, 'flagValues:\n flag3: true\n') + result = load(f) + assert result is not None and flag_keys(result) == ['flag2', 'flag3'] + + +@yaml_required +def test_reloader_measures_a_file_from_its_own_read_when_a_later_file_reads_slowly(make_fixture, tmp_path, monkeypatch): + # The file is found empty before the read of a later file stalls. Its delay counts from its + # own read, so it is over one retry delay after that read, and the retry the hold asks for + # comes then. + clock = FakeClock() + slow = os.path.join(str(tmp_path), 'slow.yaml') + write_file(slow, YAML_FLAG2) + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([f.path, slow], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + original = filedata._read_file + stall = [True] + + def read_file(path: str, skip_missing_paths: bool): + read = original(path, skip_missing_paths) + if path == slow and stall[0]: + clock.advance(0.9 * HOUR) + return read + + monkeypatch.setattr(filedata, '_read_file', read_file) + truncate(f.path) + require_held_back(f) + assert f.reloader._last_requested_retry_delay == pytest.approx(0.1 * HOUR + 0.1 * HOUR) + stall[0] = False + clock.advance(0.1 * HOUR) + result = load(f) + assert result is not None and flag_keys(result) == ['flag2'] + + +@yaml_required +def test_reloader_a_failing_other_file_does_not_restart_the_bound(make_fixture, tmp_path): + # The file keeps being written while every load finds it empty, and the other file fails to + # parse every other load. The failures do not start the file's bound over. + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + truncate(f.path) + require_held_back(f) + for i in range(1, 10): + clock.advance(0.5 * HOUR) + change_modification_time(f.path) + write_file(other, 'flagValues: [' if i % 2 == 1 else YAML_FLAG2) + assert load(f) is None + + clock.advance(0.5 * HOUR) + change_modification_time(f.path) + write_file(other, YAML_FLAG2) + result = load(f) + assert result is not None and flag_keys(result) == ['flag2'] + + +@yaml_required +def test_reloader_another_files_bound_does_not_cut_a_rewrite_short(make_fixture, tmp_path): + # One file keeps being written while every load finds it empty, until its bound is over. + # The other file is a quarter of a retry delay into an ordinary rewrite then, and stays held + # back until it is written. + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + truncate(f.path) + require_held_back(f) + for _ in range(9): + clock.advance(0.5 * HOUR) + change_modification_time(f.path) + require_held_back(f) + + clock.advance(0.25 * HOUR) + change_modification_time(f.path) + truncate(other) + require_held_back(f) + + clock.advance(0.25 * HOUR) + change_modification_time(f.path) + require_held_back(f) + + write_file(other, 'flagValues:\n flag2: false\n') + result = load(f) + assert result is not None and flag_keys(result) == ['flag2'] + assert result.flags['flag2'].variations == [False] + + +@yaml_required +def test_reloader_applies_files_emptied_in_staggered_turns_after_the_bound(make_fixture, tmp_path): + # Two files stay empty and are truncated again in turn, so that whenever one of them has + # stayed empty for the whole delay, the other is still within its delay and holds the load + # back. A file that settles so does not get a fresh bound while the other is held back. + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + truncate(f.path) + require_held_back(f) + applied = False + for i in range(1, 15): + clock.advance(54 / 60 * HOUR) + if i == 1: + truncate(other) + change_modification_time(other if i % 2 == 1 else f.path) + applied = load(f) is not None + if applied: + break + # The retry that the hold arms, a little after the other file's delay is over. + clock.advance(12 / 60 * HOUR) + applied = load(f) is not None + clock.advance(-12 / 60 * HOUR) + if applied: + break + assert applied, "the files were held back without end" + + +@yaml_required +def test_reloader_bound_of_a_file_held_back_during_a_failure_starts_over_once_it_is_read_again(make_fixture, tmp_path): + # While the other file fails to load, this file is rewritten once and its gap is held back. + # Much later, the other file is fixed while this one is in the middle of another rewrite, + # which is held back with a bound of its own. + clock = FakeClock() + broken = os.path.join(str(tmp_path), 'other.json') + write_file(broken, '{"flagValues": {"flag3": false}}') + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([f.path, broken], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + write_file(broken, '{bad') + truncate(f.path) + assert load(f) is None + write_file(f.path, YAML_FLAG2) + assert load(f) is None + clock.advance(6 * HOUR) + assert load(f) is None + assert isinstance(f.errored.get_nowait(), FileReadError) + assert f.errored.empty(), "an identical failure was reported again" + + write_file(broken, '{"flagValues": {"flag3": false}}') + truncate(f.path) + require_held_back(f) + + +@yaml_required +def test_reloader_hold_asks_for_the_retry_from_the_end_of_the_load(make_fixture, tmp_path, monkeypatch): + # The read of the file after the held one takes longer than what is left of the held file's + # delay, so the retry is due right away, after the margin. + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + truncate(f.path) + require_held_back(f) + + original = filedata._read_file + stall = [True] + + def read_file(path: str, skip_missing_paths: bool): + read = original(path, skip_missing_paths) + if path == other and stall[0]: + stall[0] = False + clock.advance(6 / 60 * HOUR) + return read + + monkeypatch.setattr(filedata, '_read_file', read_file) + clock.advance(57 / 60 * HOUR) + require_held_back(f) + assert f.reloader._last_requested_retry_delay == pytest.approx(0.1 * HOUR) + + +@pytest.mark.parametrize('name, good, broken, fixed', [ + pytest.param('data.yaml', YAML_FLAG1, 'flagValues: [', 'flagValues:\n flag1: false\n', marks=yaml_required, id='yaml'), + pytest.param('data.json', '{"flagValues": {"flag1": true}}', '{"flagValues"', '{"flagValues": {"flag1": false}}', id='json'), +]) +def test_reloader_holds_back_the_fix_of_a_file_saved_in_place_with_an_error(make_fixture, name, good, broken, fixed): + # An operator saves a typo in place, and the fix in place much later. The truncation of the + # second save is held back like the first, so the fix is applied and the file never empty. + f, clock = make_held_fixture(make_fixture, good, name=name) + truncate(f.path) + require_held_back(f) + write_file(f.path, broken) + assert load(f) is None + assert isinstance(f.errored.get_nowait(), FileReadError) + for _ in range(6): + clock.advance(HOUR) + assert load(f) is None + assert f.errored.empty(), "an identical failure was reported again" + + truncate(f.path) + require_held_back(f) + write_file(f.path, fixed) + result = load(f) + assert result is not None and result.flags['flag1'].variations == [False] + + +@yaml_required +def test_reloader_file_that_a_failing_load_did_not_reach_keeps_its_bound(make_fixture, tmp_path): + # The file is kept written while empty. A file before it fails to parse on every other load, + # so those loads never get to it, and it keeps its bound. + clock = FakeClock() + first = os.path.join(str(tmp_path), 'first.json') + write_file(first, '{"flagValues": {"flag3": true}}') + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([first, f.path], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + require_held_back(f) + for i in range(1, 10): + clock.advance(0.5 * HOUR) + change_modification_time(f.path) + write_file(first, '{"flagValues"' if i % 2 == 1 else '{"flagValues": {"flag3": true}}') + assert load(f) is None + + clock.advance(0.5 * HOUR) + change_modification_time(f.path) + write_file(first, '{"flagValues": {"flag3": true}}') + result = load(f) + assert result is not None and flag_keys(result) == ['flag3'] + + +@yaml_required +def test_reloader_load_that_holds_a_file_back_and_then_fails_does_not_start_bounds_over(make_fixture, tmp_path): + # Two files are rewritten in turn, and a third fails to parse on every other load, after one of + # the two has been held back. Those loads hold a file back, so they do not start the bounds + # over, and the files are applied in the end. + clock = FakeClock() + other = os.path.join(str(tmp_path), 'other.yaml') + last = os.path.join(str(tmp_path), 'last.json') + write_file(other, YAML_FLAG2) + write_file(last, '{"flagValues": {"flag3": true}}') + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([f.path, other, last], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + require_held_back(f) + applied = False + for i in range(1, 13): + clock.advance(0.5 * HOUR) + if i % 2 == 1: + write_file(f.path, YAML_FLAG1) + truncate(other) + write_file(last, '{"flagValues"') + else: + write_file(other, YAML_FLAG2) + truncate(f.path) + write_file(last, '{"flagValues": {"flag3": true}}') + if load(f) is not None: + applied = True + break + assert applied, "the files were held back without end" + + +@yaml_required +def test_reloader_file_found_missing_starts_a_new_delay_when_it_is_created_empty_again(make_fixture, tmp_path): + # The file is emptied and held back, then deleted, then created empty again with the + # modification time of its first emptying, while another file fails to parse, so nothing is + # applied meanwhile. Being found missing ended its empty state, so it starts a new delay. + clock = FakeClock() + broken = os.path.join(str(tmp_path), 'other.json') + write_file(broken, '{"flagValues": {"flag3": true}}') + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([f.path, broken], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, skip_missing_paths=True, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + write_file(broken, '{bad') + truncate(f.path) + first_emptying = os.stat(f.path) + assert load(f) is None + os.remove(f.path) + assert load(f) is None + + clock.advance(70 / 60 * HOUR) + write_file(f.path, '') + os.utime(f.path, ns=(first_emptying.st_atime_ns, first_emptying.st_mtime_ns)) + write_file(broken, '{"flagValues": {"flag3": true}}') + while not f.errored.empty(): + f.errored.get_nowait() + require_held_back(f) + + +@yaml_required +def test_reloader_hold_asks_for_the_retry_after_the_delay_and_a_tenth(make_fixture): + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + assert f.reloader._last_requested_retry_delay == pytest.approx(1.1 * HOUR) + + +@yaml_required +def test_reloader_later_hold_asks_for_the_rest_of_the_delay(make_fixture): + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + clock.advance(0.4 * HOUR) + require_held_back(f) + assert f.reloader._last_requested_retry_delay == pytest.approx(0.6 * HOUR + 0.1 * HOUR) + + +@yaml_required +def test_reloader_hold_asks_for_the_retry_when_the_bound_is_over(make_fixture): + # The file was written again, so its delay would end one retry delay from now, but its bound + # ends sooner. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + for _ in range(9): + clock.advance(0.5 * HOUR) + change_modification_time(f.path) + require_held_back(f) + assert f.reloader._last_requested_retry_delay == pytest.approx(0.5 * HOUR + 0.1 * HOUR) + + +@yaml_required +@pytest.mark.parametrize('due_first', ['first', 'last']) +def test_reloader_hold_asks_for_the_retry_when_the_first_held_file_is_due(make_fixture, tmp_path, due_first): + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + earlier, later = (f.path, other) if due_first == 'first' else (other, f.path) + truncate(earlier) + require_held_back(f) + clock.advance(0.5 * HOUR) + truncate(later) + require_held_back(f) + assert f.reloader._last_requested_retry_delay == pytest.approx(0.5 * HOUR + 0.1 * HOUR) + + +@yaml_required +def test_reloader_hold_after_a_change_signal_asks_for_the_same_retry(make_fixture, caplog): + caplog.set_level(logging.DEBUG) + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + + def wait_for_requested_delay(expected: float) -> None: + deadline = time.monotonic() + TEST_TIMEOUT + while f.reloader._last_requested_retry_delay != pytest.approx(expected): + assert time.monotonic() < deadline, "asked for %r" % f.reloader._last_requested_retry_delay + time.sleep(0.01) + + truncate(f.path) + f.reloader.trigger() + wait_for_deferrals(caplog, 1) + wait_for_requested_delay(1.1 * HOUR) + clock.advance(0.4 * HOUR) + f.reloader.trigger() + wait_for_deferrals(caplog, 2) + wait_for_requested_delay(0.7 * HOUR) + + +def test_reloader_failure_asks_for_the_retry_delay(make_fixture): + f = make_fixture('{"flagValues"', retry_delay=HOUR, clock=FakeClock()) + f.reloader.reload_now() + f.require_errored() + assert f.reloader._last_requested_retry_delay == HOUR + + @yaml_required def test_reloader_holds_back_a_file_that_is_emptied_again_after_its_content_was_applied(make_fixture): # A load that holds no file back ends the run, so a later rewrite is held back on its own. @@ -1358,6 +1800,23 @@ def test_reloader_a_further_hold_does_not_put_off_the_retry(make_fixture, caplog assert retry_delay <= elapsed < retry_delay + 0.65 +def test_reloader_says_why_it_loads_the_files_again(make_fixture, caplog): + caplog.set_level(logging.DEBUG) + f = make_fixture('{"flagValues": {"flag1": true}}', retry_delay=0.2) + f.reloader.reload_now() + f.require_applied() + + # The retry that a hold arms reads the files again because a file was found empty. It finds + # the file still empty, which fails as JSON, and the retry after that failure says so. + truncate(f.path) + f.reloader.trigger() + for expected in ('Reading flag data again after a file was found empty', 'Retrying flag data load after earlier failure'): + deadline = time.monotonic() + TEST_TIMEOUT + while not any(r.getMessage() == expected for r in caplog.records): + assert time.monotonic() < deadline, "no log line %r" % expected + time.sleep(0.01) + + def test_reloader_reports_an_emptied_json_file_only_at_the_retry_and_keeps_the_data(make_fixture, caplog): caplog.set_level(logging.DEBUG) f = make_fixture('{"flagValues": {"flag1": true}}', retry_delay=EMPTY_FILE_RETRY_DELAY) From c22409e1d60907d3edc77f9958236cfb6a073c91 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 3 Nov 2026 07:28:12 +0000 Subject: [PATCH 24/26] fix: Read every file before a load fails, and digest each path A load stopped at the first file that could not be read or parsed, so a file configured after it was never read and kept a bound left over from before the failure; its next in-place rewrite was then applied empty. A load now reads and classifies every configured file, then fails with the first failure, then holds files back, then merges. Every file is reached, so a load that holds nothing back starts every file's bound over. The skip-unchanged digest now covers each path and whether its file exists, not only the bytes. The format comes from the extension, so the same bytes moved from x.yaml to x.json parse differently and must not be skipped. Files in UTF-16, UTF-32 or invalid UTF-8 are reported like any other content that cannot be parsed, and the reloader keeps working; tests pin both. --- ldclient/impl/integrations/files/filedata.py | 46 ++++-- .../testing/integrations/test_filedata.py | 139 ++++++++++++++++-- 2 files changed, 157 insertions(+), 28 deletions(-) diff --git a/ldclient/impl/integrations/files/filedata.py b/ldclient/impl/integrations/files/filedata.py index 3864f659..3f752dc2 100644 --- a/ldclient/impl/integrations/files/filedata.py +++ b/ldclient/impl/integrations/files/filedata.py @@ -343,20 +343,26 @@ def _parse_files(contents: List[_FileRead], duplicate_keys_handling: DuplicateKe def _merge_files(parsed: List[Tuple[_FileRead, Optional[Document]]], duplicate_keys_handling: DuplicateKeysHandling) -> Tuple[MergeResult, bytes]: """ - Merges the parsed files. It also returns a digest of the raw file contents, so a caller can - tell whether a later load read the same bytes. One read feeds both the digest and the parse, - so the digest can never disagree with the content that was parsed. The digest covers the - bytes of the files that exist, not which file holds them. + Merges the parsed files. It also returns a digest of the files that were read, so a caller + can tell whether a later load read the same. One read feeds both the digest and the parse, + so the digest can never disagree with the content that was parsed. The digest covers each + path, whether its file exists, and its bytes: the same bytes in a file with another name can + be in another format, and parse differently. """ documents: List[Document] = [] files: List[FileSummary] = [] hasher = hashlib.sha256() for read, document in parsed: + path = read.path.encode("utf-8", errors="surrogateescape") + hasher.update(len(path).to_bytes(8, "big")) + hasher.update(path) if read.raw is None: + hasher.update(b"-") files.append(FileSummary(path=read.path)) continue + hasher.update(b"+") + hasher.update(len(read.raw).to_bytes(8, "big")) hasher.update(read.raw) - hasher.update(b"\0") documents.append(Document() if document is None else document) files.append(FileSummary(path=read.path, present=True)) @@ -433,9 +439,10 @@ class Reloader: over. Each file is held back for at most five retry delays after a load first held it back, until a load holds nothing back, so a file that keeps being rewritten is applied in the end without cutting short another file's rewrite. A file whose content is not in effect, such as a - placeholder that has always been empty, is never held back. Every file is read and parsed - before any is held back, so files emptied together wait together, and a file that cannot be - parsed fails the load even while another is held back. The clock is read after each file's + placeholder that has always been empty, is never held back. Every file is read and parsed, + even after another file fails, before any is held back, so files emptied together wait + together, and a file that cannot be read or parsed fails the load, with the first failure, + even while another is held back. The clock is read after each file's own read, so a slow read of another file does not shorten a file's delay. The worker thread starts on the first :meth:`reload_now` or :meth:`trigger` call, so a @@ -645,8 +652,11 @@ def _reload(self) -> Optional[float]: def _load(self, reached: List[str]) -> Optional[float]: """The body of :meth:`_reload`. It records in ``reached`` each file that it gets to.""" - # Every file is read, parsed and classified before any is held back, so that every file - # that is found with no entries starts its delay in this load. + # Every file is read, parsed and classified, even after another file fails, before the + # load fails, holds files back or merges them. So every file that is found with no + # entries starts its delay in this load, and every file is reached, so a load that holds + # nothing back starts every file's bound over. The load fails with the first failure. + failure: Optional[Exception] = None held_back: List[str] = [] parsed: List[Tuple[_FileRead, Optional[Document]]] = [] had_content: Dict[str, bool] = {} @@ -655,7 +665,8 @@ def _load(self, reached: List[str]) -> Optional[float]: try: read = _read_file(path, self._skip_missing_paths) except Exception as e: - return self._fail(e) + failure = failure or e + continue if read.raw is None: # A missing file is not an empty one. self._end_empty(read.path) @@ -674,7 +685,8 @@ def _load(self, reached: List[str]) -> Optional[float]: except FileReadError as e: # Content that cannot be parsed is not an empty file either. self._end_empty(read.path) - return self._fail(e) + failure = failure or e + continue has_document = document is not None if has_document: self._end_empty(read.path) @@ -690,8 +702,11 @@ def _load(self, reached: List[str]) -> Optional[float]: try: document = _parse_file(read.path, read.raw) except FileReadError as e: - return self._fail(e) + failure = failure or e + continue parsed.append((read, document)) + if failure is not None: + return self._fail(failure) if len(held_back) > 0: return self._hold_back(held_back, self._clock()) @@ -709,9 +724,8 @@ def _load(self, reached: List[str]) -> Optional[float]: # that things are good again. recovering = self._last_error_message is not None if self._skip_unchanged and not recovering and digest == self._last_good_digest: - # The digest does not say which file holds which bytes, so content that moved from - # one file to another is skipped as unchanged, and what each file contributes has to - # be remembered here too. The data in effect is confirmed, so the bounds start over. + # The data in effect is confirmed, as by an application, so it is remembered the same + # way, and the bounds start over. self._remember_applied(had_content) return None diff --git a/ldclient/testing/integrations/test_filedata.py b/ldclient/testing/integrations/test_filedata.py index 1ca6e9db..c0f14828 100644 --- a/ldclient/testing/integrations/test_filedata.py +++ b/ldclient/testing/integrations/test_filedata.py @@ -188,6 +188,32 @@ def test_parse_validates_definition_property_types(): parse_document(b'{"segments": {"seg1": {"key": "seg1", "version": 1, "included": "not a list"}}}') +@pytest.mark.parametrize('content', [ + '{"flagValues": {"flag1": true}}'.encode('utf-16'), + '{"flagValues": {"flag1": true}}'.encode('utf-32'), + b'{"flagValues": {"flag1": "\xff"}}', +], ids=['utf-16', 'utf-32', 'invalid-utf-8']) +@pytest.mark.parametrize('is_yaml', [False, pytest.param(True, marks=pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed"))], ids=['json', 'yaml']) +def test_parse_rejects_content_that_is_not_utf_8(content, is_yaml): + with pytest.raises(ValueError): + parse_document(content, is_yaml=is_yaml) + + +def test_reloader_reports_a_file_that_is_not_utf_8_and_keeps_working(make_fixture): + # A file in another encoding fails to load like any other content that cannot be parsed, and + # the reloader keeps loading the files when they change. + f = make_fixture('{"flagValues": {"flag1": true}}') + f.reloader.reload_now() + f.require_applied() + with open(f.path, 'wb') as out: + out.write('{"flagValues": {"flag1": false}}'.encode('utf-16')) + f.reloader.trigger() + assert isinstance(f.require_errored(), FileReadError) + write_file(f.path, '{"flagValues": {"flag1": false}}') + f.reloader.trigger() + assert f.require_applied().flags['flag1'].variations == [False] + + def test_read_file_json_and_errors(tmp_path): path = os.path.join(str(tmp_path), 'data.json') write_file(path, '{"flagValues": {"flag1": true}}') @@ -1392,9 +1418,9 @@ def test_reloader_holds_back_the_fix_of_a_file_saved_in_place_with_an_error(make @yaml_required -def test_reloader_file_that_a_failing_load_did_not_reach_keeps_its_bound(make_fixture, tmp_path): - # The file is kept written while empty. A file before it fails to parse on every other load, - # so those loads never get to it, and it keeps its bound. +def test_reloader_file_kept_written_while_empty_is_applied_after_its_bound_while_an_earlier_file_fails(make_fixture, tmp_path): + # The file is kept written while empty. A file before it fails to parse on every other load. + # Those loads still read and hold back this file, so they do not start its bound over. clock = FakeClock() first = os.path.join(str(tmp_path), 'first.json') write_file(first, '{"flagValues": {"flag3": true}}') @@ -1418,6 +1444,96 @@ def test_reloader_file_that_a_failing_load_did_not_reach_keeps_its_bound(make_fi assert result is not None and flag_keys(result) == ['flag3'] +@yaml_required +def test_reloader_bound_of_a_file_after_a_failing_file_starts_over_once_it_is_read_again(make_fixture, tmp_path): + # The file is held back, then a file configured before it breaks while this one gets its + # content back. The failing loads still read this file, and they hold nothing back, so its + # bound starts over. Much later the other file is fixed while this one is rewritten in place, + # and the truncation is held back. + clock = FakeClock() + first = os.path.join(str(tmp_path), 'first.json') + write_file(first, '{"flagValues": {"flag3": true}}') + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([first, f.path], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + require_held_back(f) + write_file(first, '{"flagValues"') + write_file(f.path, YAML_FLAG2) + assert load(f) is None + for _ in range(6): + clock.advance(HOUR) + assert load(f) is None + assert isinstance(f.errored.get_nowait(), FileReadError) + assert f.errored.empty() + + write_file(first, '{"flagValues": {"flag3": true}}') + truncate(f.path) + require_held_back(f) + + +@yaml_required +def test_reloader_reads_a_file_after_one_that_cannot_be_read(make_fixture, tmp_path): + # A file configured before this one goes missing, and missing files are not skipped, so it + # cannot be read. The failing loads still read this file, which got its content back, and + # they hold nothing back, so its bound starts over. + clock = FakeClock() + first = os.path.join(str(tmp_path), 'first.json') + write_file(first, '{"flagValues": {"flag3": true}}') + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([first, f.path], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + require_held_back(f) + os.remove(first) + write_file(f.path, YAML_FLAG2) + assert load(f) is None + assert f.errored.get_nowait().path == first + clock.advance(6 * HOUR) + assert load(f) is None + + write_file(first, '{"flagValues": {"flag3": true}}') + truncate(f.path) + require_held_back(f) + + +def test_reloader_failing_load_reports_the_first_failure(make_fixture, tmp_path): + # Every file is read even after one fails, and the load reports the first failure. + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues"') + write_file(second, '{"flags"') + f = make_fixture('{}') + f.reloader = Reloader([first, second], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put) + f.reloader.reload_now() + err = f.require_errored() + assert isinstance(err, FileReadError) + assert err.path == first + + +@yaml_required +def test_reloader_applies_the_same_bytes_moved_to_a_file_of_another_format(make_fixture, tmp_path): + # The format comes from the extension, so the same bytes moved from x.yaml to x.json parse + # differently: YAML reads 1e3 as a string, JSON as a number. The load is not skipped as + # unchanged. + yaml_path = os.path.join(str(tmp_path), 'x.yaml') + json_path = os.path.join(str(tmp_path), 'x.json') + content = '{"flagValues": {"flag1": 1e3}}' + write_file(yaml_path, content) + f = make_fixture('{}') + f.reloader = Reloader([yaml_path, json_path], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, skip_missing_paths=True, skip_unchanged=True) + f.reloader.reload_now() + assert f.require_applied().flags['flag1'].variations == ['1e3'] + + os.replace(yaml_path, json_path) + result = load(f) + assert result is not None and result.flags['flag1'].variations == [1000.0] + + @yaml_required def test_reloader_load_that_holds_a_file_back_and_then_fails_does_not_start_bounds_over(make_fixture, tmp_path): # Two files are rewritten in turn, and a third fails to parse on every other load, after one of @@ -1638,10 +1754,8 @@ def test_reloader_applies_a_file_that_is_deleted_and_then_created_empty_at_once( @yaml_required def test_reloader_does_not_hold_back_a_file_whose_content_moved_away(make_fixture, tmp_path): - # The content moves from one file to the other. The digest of the files' bytes does not say - # which file holds them, so it is the same as before and the load is skipped as unchanged. - # The file that no longer exists contributes no content, so when it is created empty, that - # is applied at once. + # The content moves from one file to the other. The file that no longer exists contributes no + # content, so when it is created empty, that is applied at once. clock = FakeClock() other = os.path.join(str(tmp_path), 'other.yaml') f = make_fixture(YAML_FLAG1, name='data.yaml') @@ -1651,7 +1765,8 @@ def test_reloader_does_not_hold_back_a_file_whose_content_moved_away(make_fixtur write_file(other, YAML_FLAG1) os.remove(f.path) - assert load(f) is None + result = load(f) + assert result is not None and result.files == [FileSummary(path=f.path), FileSummary(path=other, present=True, flags=1)] write_file(f.path, '') result = load(f) @@ -1661,9 +1776,8 @@ def test_reloader_does_not_hold_back_a_file_whose_content_moved_away(make_fixtur @yaml_required def test_reloader_holds_back_a_file_whose_content_was_moved_to_it_from_another(make_fixture, tmp_path): - # The file is renamed to the other configured path (mv data.yaml other.yaml), so the bytes are - # the same as before and the load is skipped as unchanged. The other path's content is in - # effect now, so an in-place rewrite of it is held back. + # The file is renamed to the other configured path (mv data.yaml other.yaml). The other + # path's content is in effect now, so an in-place rewrite of it is held back. clock = FakeClock() other = os.path.join(str(tmp_path), 'other.yaml') f = make_fixture(YAML_FLAG1, name='data.yaml') @@ -1672,7 +1786,8 @@ def test_reloader_holds_back_a_file_whose_content_was_moved_to_it_from_another(m assert flag_keys(f.require_applied()) == ['flag1'] os.replace(f.path, other) - assert load(f) is None + result = load(f) + assert result is not None and result.files == [FileSummary(path=f.path), FileSummary(path=other, present=True, flags=1)] truncate(other) require_held_back(f) From 8709890f02f4b02276e1c1dcd2f813c184dda954 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 3 Nov 2026 09:56:48 +0000 Subject: [PATCH 25/26] fix: Reject non-object JSON, read .yaml dotfiles as YAML, and pin review cases A JSON file holding null, or anything else but an object, now fails to parse, as in the other SDKs; YAML null still has no entries. A file named exactly .yaml or .yml is YAML. A parse error names the format the file's name chose. A load logs the failures after the first at Debug level. _reload returns whether it held files back, so the retry's log line does not read state outside the lock, and the digest encodes paths with os.fsencode. Tests pin the cases the reviews found unpinned: a file whose read fails starts its bound over and keeps its delay, a later failure does not replace the first, and a JSON file holding only a byte order mark is held back quietly. The docs say that YAML cut off at almost any point can still parse. Two real-clock bounds and one retry delay get wider margins. --- ldclient/impl/integrations/files/filedata.py | 96 ++++++++----- ldclient/integrations/overrides.py | 12 +- .../testing/integrations/test_filedata.py | 130 +++++++++++++++++- 3 files changed, 190 insertions(+), 48 deletions(-) diff --git a/ldclient/impl/integrations/files/filedata.py b/ldclient/impl/integrations/files/filedata.py index 3f752dc2..f8caf95d 100644 --- a/ldclient/impl/integrations/files/filedata.py +++ b/ldclient/impl/integrations/files/filedata.py @@ -155,10 +155,11 @@ def make_flag_with_value(key: str, value: Any) -> FeatureFlag: def is_yaml_path(path: str) -> bool: """ - Whether a data file is a YAML document. The file name extension decides: ``.yaml`` and - ``.yml``, in any case, are YAML, and any other file, including one with no extension, is JSON. + Whether a data file is a YAML document. The end of the file name decides: a name that ends in + ``.yaml`` or ``.yml``, in any case, is YAML, including a name that is only that, such as + ``.yaml``. Any other file, including one with no extension, is JSON. """ - return os.path.splitext(path)[1].lower() in YAML_EXTENSIONS + return os.path.basename(path).lower().endswith(tuple(YAML_EXTENSIONS)) def read_file(path: str) -> Document: @@ -196,9 +197,9 @@ def parse_document(raw: bytes, is_yaml: bool = False) -> Document: def _parse_content(raw: bytes, is_yaml: bool) -> Optional[Document]: """ - The parse behind :func:`parse_document`. Returns None when the parser reads the content as - nothing, as YAML parsers do for a file with no content or only comments, rather than as a - document, even one with no members such as ``{}``. + The parse behind :func:`parse_document`. Returns None when the YAML parser reads the content + as nothing, as it does for a file with no content, only comments, or ``null``, rather than as + a document, even one with no members such as ``{}``. A JSON document must be an object. """ text = raw.decode("utf-8-sig") if is_yaml: @@ -211,6 +212,8 @@ def _parse_content(raw: bytes, is_yaml: bool) -> Optional[Document]: # and the failure keeps the last good data until the file is complete. raise ValueError("the file has no content, which is not a JSON document") parsed = json.loads(text) + if not isinstance(parsed, dict): + raise ValueError("the document must be an object") if parsed is None: return None if not isinstance(parsed, dict): @@ -330,10 +333,11 @@ def _read_file(path: str, skip_missing_paths: bool) -> _FileRead: def _parse_file(path: str, raw: bytes) -> Optional[Document]: """Parses what was read from a file that exists. Returns None when the parser read nothing.""" + is_yaml = is_yaml_path(path) try: - return _parse_content(raw, is_yaml_path(path)) + return _parse_content(raw, is_yaml) except Exception as e: - raise FileReadError(path, "error parsing file: %s" % e) + raise FileReadError(path, "error parsing file as %s, the format of its name: %s" % ("YAML" if is_yaml else "JSON", e)) def _parse_files(contents: List[_FileRead], duplicate_keys_handling: DuplicateKeysHandling) -> Tuple[MergeResult, bytes]: @@ -353,7 +357,7 @@ def _merge_files(parsed: List[Tuple[_FileRead, Optional[Document]]], duplicate_k files: List[FileSummary] = [] hasher = hashlib.sha256() for read, document in parsed: - path = read.path.encode("utf-8", errors="surrogateescape") + path = os.fsencode(read.path) hasher.update(len(path).to_bytes(8, "big")) hasher.update(path) if read.raw is None: @@ -399,7 +403,7 @@ def _observe_all(paths: List[str]) -> List[FileState]: # A file is held back for at most this many retry delays after a load first held it back, until -# data is applied. +# a load holds nothing back. _MAX_HOLD_RETRY_DELAYS = 5 @@ -410,8 +414,8 @@ class _EmptyFile: since: Optional[float] = None """ Since when the file has had no entries without being written, on the reloader's clock, or - None when the last load that got to it read content, even content that failed to parse, or - found it missing. + None when the last load read it with content, even content that failed to parse, or found + it missing. A read that fails keeps it, as nothing is known about the content then. """ state: FileState = None """The file's state then: its modification time and size, through any symbolic link.""" @@ -419,7 +423,7 @@ class _EmptyFile: """ When a load first held the file back, or None. It bounds how long the file is held back. It is kept while loads hold files back, so files emptied in turn stay bounded, and cleared by - a load that holds nothing back and gets to the file, or when data is applied. + any load that holds nothing back, including one that fails. """ @@ -512,7 +516,8 @@ def __init__( # or one with no entries, contributed none. self._applied_had_content: Dict[str, bool] = {} # The files whose content is in effect and that loads have found with no entries. All are - # forgotten when data is applied; failures change nothing. + # forgotten when data is applied or a load is skipped as unchanged, and a load that holds + # nothing back, even one that fails, starts every file's bound over. self._empty_files: Dict[str, _EmptyFile] = {} # Whether the last load held files back, which the log line of the retry it arms names. self._held_back = False @@ -527,9 +532,9 @@ def reload_now(self) -> None: arms the same automatic retry as a failed triggered reload. """ self._ensure_started() - retry_after = self._reload() + retry_after, held = self._reload() if retry_after is not None: - self._arm_retry(retry_after, keep_sooner=True) + self._arm_retry(retry_after, keep_sooner=True, after_hold=held) def trigger(self) -> None: """ @@ -553,10 +558,11 @@ def close(self) -> None: self._closed = True self._cond.notify_all() - def _arm_retry(self, retry_after: float, keep_sooner: bool) -> None: + def _arm_retry(self, retry_after: float, keep_sooner: bool, after_hold: bool) -> None: """ Arms the automatic retry for the given time from now, unless retries are off. With - ``keep_sooner``, a retry that is already due sooner stays as it is. + ``keep_sooner``, a retry that is already due sooner stays as it is. ``after_hold`` says + that the load held files back, which the retry's log line names. """ if self._retry_delay <= 0: return @@ -567,7 +573,7 @@ def _arm_retry(self, retry_after: float, keep_sooner: bool) -> None: if keep_sooner and self._retry_deadline is not None and self._retry_deadline <= deadline: return self._retry_deadline = deadline - self._retry_after_hold = self._held_back + self._retry_after_hold = after_hold self._last_requested_retry_delay = retry_after self._cond.notify() @@ -615,48 +621,62 @@ def _run(self) -> None: else: log.info("Reloading flag data after detecting a change") try: - retry_after = self._reload() + retry_after, held = self._reload() except Exception as e: log.exception("Unexpected error while reloading flag data: %s", e) - retry_after = None + retry_after, held = None, False if retry_after is not None: - self._arm_retry(retry_after, keep_sooner=False) + self._arm_retry(retry_after, keep_sooner=False, after_hold=held) def _is_closed(self) -> bool: with self._cond: return self._closed - def _reload(self) -> Optional[float]: + def _reload(self) -> Tuple[Optional[float], bool]: """ - Performs one full load of all configured files. Returns None when the load succeeded, - including a skipped no-op application, or the time after which to load again: the retry - delay after a failure, or the time a hold lasts. + Performs one full load of all configured files. Returns the time after which to load + again, or None when the load succeeded, including a skipped no-op application: the retry + delay after a failure, or the time a hold lasts. Also returns whether the load held files + back. """ with self._reload_lock: if self._is_closed(): - return None + return None, False self._held_back = False self._holding_this_load = False reached: List[str] = [] retry_after = self._load(reached) if not self._holding_this_load: # A load that holds nothing back, whether it applies, is skipped or fails, starts the - # bound over for every file that it got to, including one that failed to be read or - # parsed: such a file is no longer held back, so its next rewrite is a new one. A - # file that the load did not get to keeps its bound. + # bound over for every file that it read or tried to read, including one that + # failed to be read or parsed: such a file is no longer held back, so its next + # rewrite is a new one. for path in reached: empty = self._empty_files.get(path) if empty is not None: empty.first_held = None - return retry_after + return retry_after, self._held_back def _load(self, reached: List[str]) -> Optional[float]: - """The body of :meth:`_reload`. It records in ``reached`` each file that it gets to.""" + """ + The body of :meth:`_reload`. It records in ``reached`` each file that it reads or tries to + read, which is every configured file, since a load reads all of them even after a failure. + """ # Every file is read, parsed and classified, even after another file fails, before the # load fails, holds files back or merges them. So every file that is found with no # entries starts its delay in this load, and every file is reached, so a load that holds # nothing back starts every file's bound over. The load fails with the first failure. failure: Optional[Exception] = None + + def note_failure(e: Exception) -> None: + nonlocal failure + if failure is None: + failure = e + else: + # The load fails with the first failure. The others are logged, so that a second + # broken file is visible too. + log.debug("Unable to load flag data, besides the first failure: %s", e) + held_back: List[str] = [] parsed: List[Tuple[_FileRead, Optional[Document]]] = [] had_content: Dict[str, bool] = {} @@ -665,7 +685,8 @@ def _load(self, reached: List[str]) -> Optional[float]: try: read = _read_file(path, self._skip_missing_paths) except Exception as e: - failure = failure or e + # Nothing is known about the content, so the file's empty state, if any, stays. + note_failure(e) continue if read.raw is None: # A missing file is not an empty one. @@ -685,7 +706,7 @@ def _load(self, reached: List[str]) -> Optional[float]: except FileReadError as e: # Content that cannot be parsed is not an empty file either. self._end_empty(read.path) - failure = failure or e + note_failure(e) continue has_document = document is not None if has_document: @@ -702,7 +723,7 @@ def _load(self, reached: List[str]) -> Optional[float]: try: document = _parse_file(read.path, read.raw) except FileReadError as e: - failure = failure or e + note_failure(e) continue parsed.append((read, document)) if failure is not None: @@ -767,9 +788,8 @@ def _should_hold_back(self, path: str, now: float) -> bool: for the whole delay is not held back. Each file is held back for at most five retry delays after a load first held it back, so that a file that keeps being written while every load finds it empty, or files that are emptied in turn, are not held back without - end. The bound starts over only at a load that holds nothing back, for every file that - load got to, including one that failed to be read or parsed, whether the load applies, is - skipped or fails. A file after the one that failed keeps its bound. A file whose content is not in effect + end. The bound starts over only at a load that holds nothing back, for every file, whether + the load applies, is skipped or fails. A load reads every file even after one fails. A file whose content is not in effect is parsed right away, and so is any file when retries are off, since nothing would read it again. """ diff --git a/ldclient/integrations/overrides.py b/ldclient/integrations/overrides.py index 7ef71954..97fe6ba4 100644 --- a/ldclient/integrations/overrides.py +++ b/ldclient/integrations/overrides.py @@ -64,8 +64,9 @@ class FileOverrideSourceBuilder(OverrideSourceBuilder): The file name extension decides the format of each file: a file whose name ends in ``.yaml`` or ``.yml``, in any case, is YAML, and any other file is JSON. YAML requires the ``pyyaml`` package. Whether a file with no content is valid depends on its format: a JSON file - with no content, or only white space, is not a JSON document and fails to load, and a YAML - file with no content is a document with no entries. + with no content, or only white space, is not a JSON document and fails to load, as does one + that holds anything but an object, such as ``null``, and a YAML file with no content, or only + comments or ``null``, is a document with no entries. When multiple files are configured, their entries are combined in the configured order. The order determines which file wins under the duplicate keys handling. A reload replaces @@ -84,9 +85,10 @@ class FileOverrideSourceBuilder(OverrideSourceBuilder): reload applies all files or none, until the file has stayed so without being written for about a second, and for at most about five seconds while it keeps being rewritten. After that, a YAML file is applied with no entries, and an empty JSON file fails to load and keeps - the overrides in effect. YAML that is cut off between two entries can still parse, so for a - YAML file, replacing its entire contents in one operation, such as renaming a fully written - temporary file over it, avoids briefly applying only a part of it. + the overrides in effect. YAML that is cut off at almost any point can still parse, with fewer + entries or with different values, so for a YAML file, replacing its entire contents in one + operation, such as renaming a fully written temporary file over it, avoids briefly applying + only a part of it. Whenever the set of overrides in effect changes, including at startup, the source logs the overrides in effect and what each configured file supplied, at Info level. diff --git a/ldclient/testing/integrations/test_filedata.py b/ldclient/testing/integrations/test_filedata.py index c0f14828..4e5454cb 100644 --- a/ldclient/testing/integrations/test_filedata.py +++ b/ldclient/testing/integrations/test_filedata.py @@ -83,7 +83,7 @@ def test_parse_json_document(): assert document.segments['seg1'].included == {'user1'} -@pytest.mark.parametrize('path', ['/data.yaml', '/data.yml', '/data.YAML', '/data.Yml', 'relative/data.yaml']) +@pytest.mark.parametrize('path', ['/data.yaml', '/data.yml', '/data.YAML', '/data.Yml', 'relative/data.yaml', '/x/.yaml', '/x/.YML']) def test_a_yaml_extension_selects_yaml(path): assert is_yaml_path(path) @@ -154,6 +154,19 @@ def test_parse_fills_in_missing_key_and_version(): assert document.segments['seg1'].version == 1 +@pytest.mark.parametrize('content', [b'null', b' null\n', b'["a", "b"]', b'1', b'"text"', b'true']) +def test_parse_rejects_a_json_document_that_is_not_an_object(content): + with pytest.raises(ValueError) as excinfo: + parse_document(content) + assert str(excinfo.value) == 'the document must be an object' + + +@pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed") +def test_parse_reads_yaml_null_as_no_entries(): + assert parse_document(b'null\n', is_yaml=True) == Document() + assert parse_document(b'~\n', is_yaml=True) == Document() + + def test_parse_rejects_documents_that_are_not_objects(): if have_yaml: with pytest.raises(ValueError): @@ -662,7 +675,9 @@ def test_reloader_retries_after_failed_initial_load(make_fixture): def test_reloader_stops_retrying_after_success(make_fixture): - f = make_fixture('{"flagValues"', retry_delay=SHORT_DELAY, skip_unchanged=True) + # The retry delay is well over the pause of the fixture's in-place writes, so that a retry is + # unlikely to read the file in the middle of one on a slow machine. + f = make_fixture('{"flagValues"', retry_delay=0.2, skip_unchanged=True) f.reloader.reload_now() f.require_errored() @@ -1501,6 +1516,110 @@ def test_reloader_reads_a_file_after_one_that_cannot_be_read(make_fixture, tmp_p require_held_back(f) +@yaml_required +def test_reloader_starts_the_bound_of_a_file_that_cannot_be_read_over(make_fixture, monkeypatch): + # The file is held back, then cannot be read for a long time, then is readable again and + # empty after a new write. The failing loads hold nothing back and tried to read the file, + # so its bound started over, and the new emptying is held back. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + + original = filedata._read_file + unreadable = [True] + + def read_file(path: str, skip_missing_paths: bool): + if path == f.path and unreadable[0]: + raise FileReadError(path, "unable to read file: permission denied") + return original(path, skip_missing_paths) + + monkeypatch.setattr(filedata, '_read_file', read_file) + for _ in range(6): + assert load(f) is None + clock.advance(HOUR) + assert isinstance(f.errored.get_nowait(), FileReadError) + + unreadable[0] = False + change_modification_time(f.path) + require_held_back(f) + + +@yaml_required +def test_reloader_keeps_the_delay_of_a_held_back_file_through_a_failed_read(make_fixture, monkeypatch): + # A read that fails says nothing about the content, so the file's delay goes on. Once it can + # be read again, still empty and unmodified, it has been empty for the whole delay. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + + original = filedata._read_file + unreadable = [True] + + def read_file(path: str, skip_missing_paths: bool): + if path == f.path and unreadable[0]: + raise FileReadError(path, "unable to read file: permission denied") + return original(path, skip_missing_paths) + + monkeypatch.setattr(filedata, '_read_file', read_file) + clock.advance(0.5 * HOUR) + assert load(f) is None + assert isinstance(f.errored.get_nowait(), FileReadError) + + unreadable[0] = False + clock.advance(0.5 * HOUR) + result = load(f) + assert result is not None and result.flags == {} + + +def test_reloader_holds_back_a_json_file_that_holds_only_a_byte_order_mark(make_fixture, caplog): + # A byte order mark is not content, so a JSON file in effect that holds only one, as a read + # can see of a file being written, is held back quietly. + caplog.set_level(logging.DEBUG) + f, clock = make_held_fixture(make_fixture, '{"flagValues": {"flag1": true}}', name='data.json') + with open(f.path, 'wb') as out: + out.write(b'\xef\xbb\xbf') + require_held_back(f) + assert deferrals_logged(caplog) == 1 + + +def test_reloader_reports_a_json_file_that_is_not_an_object(make_fixture): + f, clock = make_held_fixture(make_fixture, '{"flagValues": {"flag1": true}}', name='data.json') + write_file(f.path, 'null') + assert load(f) is None + err = f.errored.get_nowait() + assert isinstance(err, FileReadError) + assert 'must be an object' in str(err) + assert 'as JSON, the format of its name' in str(err) + + +def test_reloader_reports_the_first_failure_when_an_empty_file_also_fails(make_fixture, tmp_path): + # The first file cannot be parsed, and the second is empty, which as JSON fails too. The load + # reports the first failure, and logs the other at Debug level. + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues"') + write_file(second, '') + f = make_fixture('{}') + f.reloader = Reloader([first, second], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put) + f.reloader.reload_now() + err = f.require_errored() + assert isinstance(err, FileReadError) + assert err.path == first + + +def test_reloader_logs_the_failures_after_the_first(make_fixture, tmp_path, caplog): + caplog.set_level(logging.DEBUG) + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues"') + write_file(second, '{"flags"') + f = make_fixture('{}') + f.reloader = Reloader([first, second], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put) + f.reloader.reload_now() + assert f.require_errored().path == first + assert any(r.levelno == logging.DEBUG and second in r.getMessage() for r in caplog.records) + + def test_reloader_failing_load_reports_the_first_failure(make_fixture, tmp_path): # Every file is read even after one fails, and the load reports the first failure. first = os.path.join(str(tmp_path), 'first.json') @@ -1888,7 +2007,7 @@ def test_reloader_applies_an_emptied_yaml_file_that_stays_empty_at_the_retry(mak # retry delay, so that first retry applies it as a document with no entries. assert f.require_applied().flags == {} elapsed = time.monotonic() - started - assert EMPTY_FILE_RETRY_DELAY <= elapsed < 1.5 * EMPTY_FILE_RETRY_DELAY + assert EMPTY_FILE_RETRY_DELAY <= elapsed < 2 * EMPTY_FILE_RETRY_DELAY assert deferrals_logged(caplog) == 1 @@ -1906,13 +2025,14 @@ def test_reloader_a_further_hold_does_not_put_off_the_retry(make_fixture, caplog started = time.monotonic() f.reloader.trigger() wait_for_deferrals(caplog, 1) - time.sleep(1.0) + time.sleep(1.5) f.reloader.trigger() wait_for_deferrals(caplog, 2) + # A whole retry delay from the second hold would come at 4.5 s. assert f.require_applied().flags == {} elapsed = time.monotonic() - started - assert retry_delay <= elapsed < retry_delay + 0.65 + assert retry_delay <= elapsed < retry_delay + 1.0 def test_reloader_says_why_it_loads_the_files_again(make_fixture, caplog): From 215e0c99af5a0e3ea8bb5dff0fac388b784a63b5 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 3 Nov 2026 23:36:14 +0000 Subject: [PATCH 26/26] docs: Say only what an application needs about rewriting override files The builder docs described the hold's delay, its bound and the other files waiting. They now say that the file name decides the format, that files must be UTF-8, that files can be rewritten in place, how to remove a file's overrides, and that a YAML file is best replaced in one operation because a partly written one can still parse. The mechanics stay in internal comments. --- ldclient/integrations/overrides.py | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/ldclient/integrations/overrides.py b/ldclient/integrations/overrides.py index 97fe6ba4..0b35cb3e 100644 --- a/ldclient/integrations/overrides.py +++ b/ldclient/integrations/overrides.py @@ -61,12 +61,9 @@ class FileOverrideSourceBuilder(OverrideSourceBuilder): ``client_side_only=True`` leaves it out, even when the flag it overrides is available to them. To override such a flag, define it in ``flags`` with ``"clientSide": true``. - The file name extension decides the format of each file: a file whose name ends in - ``.yaml`` or ``.yml``, in any case, is YAML, and any other file is JSON. YAML requires the - ``pyyaml`` package. Whether a file with no content is valid depends on its format: a JSON file - with no content, or only white space, is not a JSON document and fails to load, as does one - that holds anything but an object, such as ``null``, and a YAML file with no content, or only - comments or ``null``, is a document with no entries. + The file name decides the format of each file: a file whose name ends in ``.yaml`` or + ``.yml`` is YAML, which requires the ``pyyaml`` package, and any other file is JSON. Files + must be UTF-8. When multiple files are configured, their entries are combined in the configured order. The order determines which file wins under the duplicate keys handling. A reload replaces @@ -77,18 +74,12 @@ class FileOverrideSourceBuilder(OverrideSourceBuilder): source logs the failure, retries after a short delay, and recovers on its own once the file is readable again. - A file can be rewritten in place. To remove a file's overrides, delete it or write a document - with no entries, such as ``{}``. A JSON file with no content is not a valid document: it fails - to load, and the overrides in effect stay. A file whose content is in effect and that is found - empty, or with only a YAML comment or document marker, may be in the middle of a rewrite. The - source then keeps the overrides in effect, and changes to the other files wait too, since a - reload applies all files or none, until the file has stayed so without being written for - about a second, and for at most about five seconds while it keeps being rewritten. After - that, a YAML file is applied with no entries, and an empty JSON file fails to load and keeps - the overrides in effect. YAML that is cut off at almost any point can still parse, with fewer - entries or with different values, so for a YAML file, replacing its entire contents in one - operation, such as renaming a fully written temporary file over it, avoids briefly applying - only a part of it. + Files can be rewritten in place: if the source finds a file empty while it is being + rewritten, it keeps the overrides in effect until the new content is written. To remove a + file's overrides, delete the file or write a document with no entries, such as ``{}``. An + empty JSON file is not a valid document. A partly written YAML file can still parse, so for a + YAML file, replacing its contents in one operation, such as by renaming a fully written + temporary file over it, avoids briefly applying a partly written file. Whenever the set of overrides in effect changes, including at startup, the source logs the overrides in effect and what each configured file supplied, at Info level.