diff --git a/lib/sdk/server/build.gradle b/lib/sdk/server/build.gradle index 1dadb4ed..4a2c1537 100644 --- a/lib/sdk/server/build.gradle +++ b/lib/sdk/server/build.gradle @@ -70,8 +70,8 @@ ext.versions = [ "gson": "2.13.1", "guava": "32.0.1-jre", "jackson": "2.11.2", - "launchdarklyJavaSdkCommon": "2.3.0", - "launchdarklyJavaSdkInternal": "1.11.1", + "launchdarklyJavaSdkCommon": "2.6.0", + "launchdarklyJavaSdkInternal": "1.13.0", "launchdarklyLogging": "1.1.0", "okhttp": "4.12.0", // specify this for the SDK build instead of relying on the transitive dependency from okhttp-eventsource "okhttpEventsource": "5.0.0", diff --git a/lib/sdk/server/contract-tests/service/src/main/java/sdktest/Representations.java b/lib/sdk/server/contract-tests/service/src/main/java/sdktest/Representations.java index cd7593e7..a2e03b1c 100644 --- a/lib/sdk/server/contract-tests/service/src/main/java/sdktest/Representations.java +++ b/lib/sdk/server/contract-tests/service/src/main/java/sdktest/Representations.java @@ -34,6 +34,21 @@ public static class SdkConfigParams { SdkConfigServiceEndpointParams serviceEndpoints; SdkConfigHookParams hooks; SdkConfigDataSystemParams dataSystem; + SdkConfigOverridesParams overrides; + } + + /** + * Configuration for the SDK's file-based flag overrides (the "flag-overrides" capability). + */ + public static class SdkConfigOverridesParams { + /** Absolute paths of the override files, in precedence order. */ + String[] filePaths; + /** "fail" (the default) or "ignore". */ + String duplicateKeysHandling; + /** "polling" or "watching"; null means the SDK default. */ + String changeDetection; + /** The polling interval in milliseconds; the SDK may raise it to its minimum. */ + Long pollIntervalMs; } public static class SdkConfigStreamParams { diff --git a/lib/sdk/server/contract-tests/service/src/main/java/sdktest/SdkClientEntity.java b/lib/sdk/server/contract-tests/service/src/main/java/sdktest/SdkClientEntity.java index f6bb9d96..53da61ac 100644 --- a/lib/sdk/server/contract-tests/service/src/main/java/sdktest/SdkClientEntity.java +++ b/lib/sdk/server/contract-tests/service/src/main/java/sdktest/SdkClientEntity.java @@ -32,6 +32,9 @@ import com.launchdarkly.sdk.server.integrations.FDv2PollingInitializerBuilder; import com.launchdarkly.sdk.server.integrations.FDv2PollingSynchronizerBuilder; import com.launchdarkly.sdk.server.integrations.FDv2StreamingSynchronizerBuilder; +import com.launchdarkly.sdk.server.integrations.FileData; +import com.launchdarkly.sdk.server.integrations.FileOverrideSourceBuilder; +import com.launchdarkly.sdk.server.integrations.FileOverrides; import com.launchdarkly.sdk.server.interfaces.BigSegmentStoreStatusProvider; import com.launchdarkly.sdk.server.subsystems.DataSourceBuilder; import com.launchdarkly.sdk.server.datasources.Initializer; @@ -68,6 +71,7 @@ import sdktest.Representations.SdkConfigParams; import sdktest.Representations.SdkConfigDataSystemParams; import sdktest.Representations.SdkConfigDataInitializerParams; +import sdktest.Representations.SdkConfigOverridesParams; import sdktest.Representations.SdkConfigSynchronizerParams; import sdktest.Representations.SdkConfigPollingParams; import sdktest.Representations.SdkConfigStreamingParams; @@ -587,12 +591,53 @@ private LDConfig buildSdkConfig(SdkConfigParams params, String tag) { dataSystemBuilder.fDv1FallbackSynchronizer(fdv1Polling); } + if (params.overrides != null) { + dataSystemBuilder.overrides(makeOverridesConfig(params.overrides)); + } + builder.dataSystem(dataSystemBuilder); + } else if (params.overrides != null) { + throw new IllegalArgumentException("flag overrides require the data system to be configured"); } return builder.build(); } + private static FileOverrideSourceBuilder makeOverridesConfig(SdkConfigOverridesParams params) { + FileOverrideSourceBuilder overrides = FileOverrides.source(); + if (params.filePaths != null) { + overrides.filePaths(params.filePaths); + } + if (params.duplicateKeysHandling != null) { + switch (params.duplicateKeysHandling) { + case "fail": + overrides.duplicateKeysHandling(FileData.DuplicateKeysHandling.FAIL); + break; + case "ignore": + overrides.duplicateKeysHandling(FileData.DuplicateKeysHandling.IGNORE); + break; + default: + throw new IllegalArgumentException("unknown duplicate keys handling: " + params.duplicateKeysHandling); + } + } + if (params.changeDetection != null) { + switch (params.changeDetection) { + case "polling": + overrides.changeDetection(FileOverrides.ChangeDetection.POLLING); + break; + case "watching": + overrides.changeDetection(FileOverrides.ChangeDetection.WATCHING); + break; + default: + throw new IllegalArgumentException("unknown change detection mode: " + params.changeDetection); + } + } + if (params.pollIntervalMs != null) { + overrides.pollInterval(Duration.ofMillis(params.pollIntervalMs)); + } + return overrides; + } + private DataSourceBuilder createSynchronizer( SdkConfigSynchronizerParams synchronizer, String payloadFilter) { diff --git a/lib/sdk/server/contract-tests/service/src/main/java/sdktest/TestService.java b/lib/sdk/server/contract-tests/service/src/main/java/sdktest/TestService.java index 45eeead7..51f9fb4e 100644 --- a/lib/sdk/server/contract-tests/service/src/main/java/sdktest/TestService.java +++ b/lib/sdk/server/contract-tests/service/src/main/java/sdktest/TestService.java @@ -21,7 +21,7 @@ import sdktest.Representations.Status; public class TestService { - private static final int PORT = 8000; + private static final int DEFAULT_PORT = 8000; private static final String[] CAPABILITIES = new String[]{ "all-flags-client-side-only", "all-flags-details-only-for-tracked-flags", @@ -47,7 +47,8 @@ public class TestService { "fdv1-fallback", "instance-id", "retry-conformance-fdv1-streaming", - "retry-conformance-fdv1-polling" + "retry-conformance-fdv1-polling", + "flag-overrides" }; static final Gson gson = new GsonBuilder().serializeNulls().create(); @@ -83,10 +84,15 @@ public static void main(String[] args) throws Exception { .addRegex("POST", Pattern.compile("/clients/(.*)"), ctx -> service.postClientCommand(diableKeepAlive(ctx))) .addRegex("DELETE", Pattern.compile("/clients/(.*)"), ctx -> service.deleteClient(diableKeepAlive(ctx))); - HttpServer server = HttpServer.start(PORT, router); + int port = DEFAULT_PORT; + String portEnv = System.getenv("PORT"); + if (portEnv != null && !portEnv.isEmpty()) { + port = Integer.parseInt(portEnv); + } + HttpServer server = HttpServer.start(port, router); server.getRecorder().setEnabled(false); // don't accumulate a request log - System.out.println("Listening on port " + PORT); + System.out.println("Listening on port " + port); // need to explicitly sleep because HttpServer now starts as a daemon thread while (true) { diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataModel.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataModel.java index 1e919cfe..0390abd2 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataModel.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataModel.java @@ -145,6 +145,15 @@ static final class FeatureFlag implements VersionedData, JsonHelpers.PostProcess private Migration migration; private boolean excludeFromSummaries; + // True if this definition came from the SDK's override store rather than from LaunchDarkly. The + // field is transient so that it never appears in the JSON form of the flag and never takes part + // in deserialization. Only the override layer sets it, on a copy that it owns. Evaluation reads + // it to mark the evaluation as override-affected. Other readers treat a marked flag the same as + // any other flag. + // + // Flag overrides are currently experimental and subject to change. + private transient boolean isOverride; + /** * Container for migration specific flag data. */ @@ -267,6 +276,31 @@ boolean isExcludeFromSummaries() { return excludeFromSummaries; } + /** + * Returns true if this definition came from the override store. + * + * @return true for an override entry + */ + boolean isOverride() { + return isOverride; + } + + /** + * Returns a shallow copy of this flag that carries the override marker. The copy shares its + * nested lists and its preprocessing data with this flag and never writes to them. This flag is + * not modified. + * + * @return a marked copy + */ + FeatureFlag markedAsOverride() { + FeatureFlag copy = new FeatureFlag(key, version, on, prerequisites, salt, targets, contextTargets, rules, + fallthrough, offVariation, variations, clientSide, trackEvents, trackEventsFallthrough, + debugEventsUntilDate, deleted, samplingRatio, migration, excludeFromSummaries); + copy.preprocessed = preprocessed; + copy.isOverride = true; + return copy; + } + public void afterDeserialized() { DataModelPreprocessing.preprocessFlag(this); } @@ -507,6 +541,10 @@ static final class Segment implements VersionedData, JsonHelpers.PostProcessingD private ContextKind unboundedContextKind; private Integer generation; + // True if this definition came from the SDK's override store rather than from LaunchDarkly. See + // the note on the same field in FeatureFlag. + private transient boolean isOverride; + Segment() {} Segment(String key, @@ -588,6 +626,28 @@ public Integer getGeneration() { return generation; } + /** + * Returns true if this definition came from the override store. + * + * @return true for an override entry + */ + boolean isOverride() { + return isOverride; + } + + /** + * Returns a shallow copy of this segment that carries the override marker. The copy shares its + * nested collections with this segment and never writes to them. This segment is not modified. + * + * @return a marked copy + */ + Segment markedAsOverride() { + Segment copy = new Segment(key, included, excluded, includedContexts, excludedContexts, salt, rules, + version, deleted, unbounded, unboundedContextKind, generation); + copy.isOverride = true; + return copy; + } + public void afterDeserialized() { DataModelPreprocessing.preprocessSegment(this); } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataModelDependencies.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataModelDependencies.java index 17ca660e..543755d5 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataModelDependencies.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataModelDependencies.java @@ -218,6 +218,14 @@ public int compare(DataKind o1, DataKind o2) { /** * Maintains a bidirectional dependency graph that can be updated whenever an item has changed. */ + /** + * Dependency edges that live outside the LaunchDarkly data, such as those of the override + * layer's entries. Given an item, adds it and everything that depends on it through those edges. + */ + interface ExternalDependencies { + void addAffectedItems(Set itemsOut, KindAndKey item); + } + static final class DependencyTracker { private final Map> dependenciesFrom = new HashMap<>(); private final Map> dependenciesTo = new HashMap<>(); diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceUpdatesImpl.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceUpdatesImpl.java index 2f945f64..e9d7854f 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceUpdatesImpl.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceUpdatesImpl.java @@ -29,6 +29,9 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Deque; +import java.util.Arrays; +import java.util.ArrayDeque; import java.util.Set; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; @@ -56,6 +59,10 @@ final class DataSourceUpdatesImpl implements DataSourceUpdateSink, DataSourceUpd private final EventBroadcasterImpl flagChangeEventNotifier; private final EventBroadcasterImpl dataSourceStatusNotifier; private final DataModelDependencies.DependencyTracker dependencyTracker = new DataModelDependencies.DependencyTracker(); + // Dependencies of definitions the data system does not hold. The override layer supplies its + // entries' dependencies, so that a change to a LaunchDarkly item also notifies the overridden + // flags that reference it. Null when no override source is configured. + private volatile DataModelDependencies.ExternalDependencies externalDependencies = null; private final DataStoreStatusProvider dataStoreStatusProvider; private final OutageTracker outageTracker; private final Object stateLock = new Object(); @@ -84,6 +91,47 @@ final class DataSourceUpdatesImpl implements DataSourceUpdateSink, DataSourceUpd currentStatus = new Status(State.INITIALIZING, Instant.now(), null); } + /** + * Registers dependency edges that live outside the LaunchDarkly data, such as the override + * layer's. Called once during construction of the data system. + */ + void setExternalDependencies(DataModelDependencies.ExternalDependencies externalDependencies) { + this.externalDependencies = externalDependencies; + } + + /** + * Adds the modified item and everything that depends on it to the set. The LaunchDarkly data and + * the external dependencies each know their own edges, and a chain can cross between them in + * either direction: an overridden flag can reference a LaunchDarkly segment, and a LaunchDarkly + * flag can have an overridden prerequisite. Each newly reached item is therefore expanded through + * both until nothing new is found. + */ + private void addAffectedItems(Set itemsOut, KindAndKey item) { + DataModelDependencies.ExternalDependencies external = externalDependencies; + if (external == null) { + dependencyTracker.addAffectedItems(itemsOut, item); + return; + } + Deque pending = new ArrayDeque<>(); + pending.push(item); + while (!pending.isEmpty()) { + KindAndKey next = pending.pop(); + // Each tracker gets its own set: a tracker skips an item that is already in the set it is + // given, so sharing one set would stop the second tracker at the starting item. + Set viaData = new HashSet<>(); + dependencyTracker.addAffectedItems(viaData, next); + Set viaExternal = new HashSet<>(); + external.addAffectedItems(viaExternal, next); + for (Set reached : Arrays.asList(viaData, viaExternal)) { + for (KindAndKey reachedItem : reached) { + if (itemsOut.add(reachedItem)) { + pending.push(reachedItem); + } + } + } + } + } + @Override public boolean init(FullDataSet allData) { Map> oldData = null; @@ -132,7 +180,7 @@ public boolean upsert(DataKind kind, String key, ItemDescriptor item) { dependencyTracker.updateDependenciesFrom(kind, key, item); if (hasFlagChangeEventListeners()) { Set affectedItems = new HashSet<>(); - dependencyTracker.addAffectedItems(affectedItems, new KindAndKey(kind, key)); + addAffectedItems(affectedItems, new KindAndKey(kind, key)); sendChangeEvents(affectedItems); } } @@ -270,7 +318,7 @@ private Set computeChangedItemsForFullDataSet(Map handlePartialChangeset( for (Map.Entry itemEntry: kindEntry.getValue().getItems()) { String key = itemEntry.getKey(); dependencyTracker.updateDependenciesFrom(kind, key, itemEntry.getValue()); - dependencyTracker.addAffectedItems(affectedItems, new KindAndKey(kind, key)); + addAffectedItems(affectedItems, new KindAndKey(kind, key)); } } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSystem.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSystem.java index 6558cc45..1eb6f9bd 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSystem.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSystem.java @@ -62,6 +62,15 @@ interface DataSystem { * @return the environment ID, or null */ String getEnvironmentId(); + + /** + * Returns the override layer, or null if no override source is configured. The layer holds the + * entries that take precedence over LaunchDarkly data; {@link #getStore()} already applies them. + * The client consults the layer directly only for the not-initialized short-circuit. + * + * @return the override layer, or null + */ + OverrideLayer getOverrideLayer(); } /** diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DefaultEventProcessorWrapper.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DefaultEventProcessorWrapper.java index 472c4bb6..aa25bf29 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DefaultEventProcessorWrapper.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DefaultEventProcessorWrapper.java @@ -34,6 +34,15 @@ public void recordEvaluationEvent(LDContext context, String flagKey, int flagVer LDValue value, EvaluationReason reason, LDValue defaultValue, String prerequisiteOfFlagKey, boolean requireFullEvent, Long debugEventsUntilDate, boolean excludeFromSummaries, Long samplingRatio) { + recordEvaluationEvent(context, flagKey, flagVersion, variation, value, reason, defaultValue, prerequisiteOfFlagKey, + requireFullEvent, debugEventsUntilDate, excludeFromSummaries, samplingRatio, false); + } + + @Override + public void recordEvaluationEvent(LDContext context, String flagKey, int flagVersion, int variation, + LDValue value, EvaluationReason reason, LDValue defaultValue, String prerequisiteOfFlagKey, + boolean requireFullEvent, Long debugEventsUntilDate, boolean excludeFromSummaries, + Long samplingRatio, boolean overrideAffected) { eventProcessor.sendEvent(new Event.FeatureRequest( System.currentTimeMillis(), flagKey, @@ -48,7 +57,8 @@ public void recordEvaluationEvent(LDContext context, String flagKey, int flagVer debugEventsUntilDate, false, samplingRatio != null ? samplingRatio : 1, - excludeFromSummaries + excludeFromSummaries, + overrideAffected )); } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/EvalResult.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/EvalResult.java index 8611afbb..ac7a2b89 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/EvalResult.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/EvalResult.java @@ -228,6 +228,14 @@ public EvaluationDetail getAsString() { */ public boolean isForceReasonTracking() { return forceReasonTracking; } + /** + * Returns true if an override affected this evaluation, directly or transitively. The value is + * the reason's indicator, so the result and the reason it returns to the caller always agree. + * Flag overrides are currently experimental and subject to change. + * @return true if an override affected the evaluation + */ + public boolean isOverrideAffected() { return anyType.getReason().isOverrideAffected(); } + public List getPrerequisiteEvalRecords() { return prerequisiteEvalRecords; } /** @@ -251,6 +259,17 @@ public EvalResult withForceReasonTracking(boolean newValue) { public EvalResult withPrerequisiteEvalRecords(List newValue) { return this.prerequisiteEvalRecords == newValue ? this : new EvalResult(this, newValue); } + + /** + * Returns a transformed copy of this EvalResult whose reason carries the given override-affected + * indicator, or this same instance if the indicator is unchanged. Precomputed results are shared + * between evaluations, so a marked result is always a new instance. + * @param newValue the new value for the indicator + * @return a transformed copy + */ + public EvalResult withOverrideAffected(boolean newValue) { + return withReason(anyType.getReason().withOverrideAffected(newValue)); + } @Override public boolean equals(Object other) { diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/EvaluationRecorder.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/EvaluationRecorder.java index 8dfd1005..75a16342 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/EvaluationRecorder.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/EvaluationRecorder.java @@ -16,7 +16,8 @@ default void recordEvaluation(DataModel.FeatureFlag flag, LDContext context, Eva default void recordPrerequisiteEvaluation(DataModel.FeatureFlag flag, DataModel.FeatureFlag prereqOfFlag, LDContext context, EvalResult result) { // default is no op } - default void recordEvaluationError(DataModel.FeatureFlag flag, LDContext context, LDValue defaultValue, EvaluationReason.ErrorKind errorKind) { + default void recordEvaluationError(DataModel.FeatureFlag flag, LDContext context, LDValue defaultValue, + EvaluationReason.ErrorKind errorKind, boolean overrideAffected) { // default is no op } default void recordEvaluationUnknownFlagError(String flagKey, LDContext context, LDValue defaultValue, EvaluationReason.ErrorKind errorKind) { diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/Evaluator.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/Evaluator.java index 845d4aaf..fefd1add 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/Evaluator.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/Evaluator.java @@ -124,6 +124,12 @@ private static class EvaluatorState { private List prerequisiteStack = null; private List prerequisiteEvalRecords = new ArrayList<>(0); // 0 initial capacity uses a static instance for performance private List segmentStack = null; + // True if the current evaluation scope has read a definition that carries the override marker. + // The scope starts from its own flag's marker. Each segment read can set it. Around a + // prerequisite evaluation the value is saved and reset, so the prerequisite's record reflects + // only the definitions that its own subtree read, and the parent scope accumulates that result + // afterwards. The marking therefore propagates upward only. + private boolean overrideAffected = false; } Evaluator(Getters getters, LDLogger logger) { @@ -146,6 +152,8 @@ EvalResult evaluate(FeatureFlag flag, LDContext context, @Nonnull EvaluationReco EvaluatorState state = new EvaluatorState(); state.originalFlag = flag; + // Reading the flag's own definition is the first read of this scope. + state.overrideAffected = flag.isOverride(); try { EvalResult result = evaluateInternal(flag, context, recorder, state); @@ -160,10 +168,12 @@ EvalResult evaluate(FeatureFlag flag, LDContext context, @Nonnull EvaluationReco result = result.withPrerequisiteEvalRecords(state.prerequisiteEvalRecords); } - return result; + return result.withOverrideAffected(state.overrideAffected); } catch (EvaluationException e) { logger.error("Could not evaluate flag \"{}\": {}", flag.getKey(), e.getMessage()); - return EvalResult.error(e.errorKind); + // An error result is marked too. A malformed override definition yields the caller's default + // value with an error reason, and an override still affected that result. + return EvalResult.error(e.errorKind).withOverrideAffected(state.overrideAffected); } } @@ -246,7 +256,20 @@ private EvalResult checkPrerequisites(FeatureFlag flag, LDContext context, @Nonn logger.error("Could not retrieve prerequisite flag \"{}\" when evaluating \"{}\"", prereq.getKey(), flag.getKey()); prereqOk = false; } else { - EvalResult prereqEvalResult = evaluateInternal(prereqFeatureFlag, context, recorder, state); + // The prerequisite evaluation is a scope of its own. Its marking starts from its own flag's + // marker, so its record reflects only the definitions that its subtree read. This scope + // accumulates that result afterwards, whether the nested evaluation returns or throws. + boolean parentOverrideAffected = state.overrideAffected; + state.overrideAffected = prereqFeatureFlag.isOverride(); + EvalResult prereqEvalResult; + boolean prereqOverrideAffected; + try { + prereqEvalResult = evaluateInternal(prereqFeatureFlag, context, recorder, state); + } finally { + prereqOverrideAffected = state.overrideAffected; + state.overrideAffected = parentOverrideAffected || prereqOverrideAffected; + } + prereqEvalResult = prereqEvalResult.withOverrideAffected(prereqOverrideAffected); // Note that if the prerequisite flag is off, we don't consider it a match no matter what its // off variation was. But we still need to evaluate it in order to generate an event. if (!prereqFeatureFlag.isOn() || prereqEvalResult.getVariationIndex() != prereq.getVariation()) { @@ -461,6 +484,12 @@ private boolean matchAnySegment(List values, LDContext context, Evaluat } Segment segment = getters.getSegment(segmentKey); if (segment != null) { + // The segment definition is read at this point, so an override segment marks the scope here. + // A match is not required. A negated clause turns a non-match into a match, so the definition + // shapes the result either way. + if (segment.isOverride()) { + state.overrideAffected = true; + } if (segmentMatchesContext(segment, context, state)) { return true; } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv1DataSystem.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv1DataSystem.java index a03da864..92020402 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv1DataSystem.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv1DataSystem.java @@ -160,6 +160,11 @@ public String getEnvironmentId() { return dataStore.getEnvironmentId(); } + @Override + public OverrideLayer getOverrideLayer() { + return null; + } + @Override public void close() throws IOException { if (disposed) { diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java index 5053625b..bafad08f 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java @@ -2,6 +2,7 @@ import com.google.common.collect.ImmutableList; import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.logging.LogValues; import com.launchdarkly.sdk.server.datasources.Initializer; import com.launchdarkly.sdk.server.datasources.SelectorSource; import com.launchdarkly.sdk.server.datasources.Synchronizer; @@ -15,6 +16,7 @@ import com.launchdarkly.sdk.server.subsystems.DataStore; import com.launchdarkly.sdk.server.subsystems.LoggingConfiguration; import com.launchdarkly.sdk.server.subsystems.DataSystemConfiguration; +import com.launchdarkly.sdk.server.subsystems.OverrideSource; import java.io.Closeable; import java.io.IOException; @@ -34,6 +36,15 @@ final class FDv2DataSystem implements DataSystem, Closeable { private final FlagChangeNotifier flagChanged; private final DataSourceStatusProvider dataSourceStatusProvider; private final DataStoreStatusProvider dataStoreStatusProvider; + private final LDLogger logger; + // The following are non-null only when an override source is configured. The layer holds the + // override entries. The overlay serves them in preference to the store's data at the store read + // boundary. The source populates the layer at runtime. None of them take part in + // initialization or data source status. + private final OverrideLayer overrideLayer; + private final OverrideSinkImpl overrideSink; + // Null after the source failed to start or was closed. + private volatile OverrideSource overrideSource; private boolean disposed = false; private FDv2DataSystem( @@ -41,14 +52,29 @@ private FDv2DataSystem( DataSource dataSource, DataSourceStatusProvider dataSourceStatusProvider, DataStoreStatusProvider dataStoreStatusProvider, - FlagChangeNotifier flagChanged + FlagChangeNotifier flagChanged, + EventBroadcasterImpl flagChangeBroadcaster, + OverrideSource overrideSource, + LDLogger logger ) { this.store = store; this.dataSource = dataSource; this.dataStoreStatusProvider = dataStoreStatusProvider; this.dataSourceStatusProvider = dataSourceStatusProvider; this.flagChanged = flagChanged; - this.readOnlyStore = new ReadonlyStoreFacade(store); + this.logger = logger; + ReadOnlyStore baseStore = new ReadonlyStoreFacade(store); + this.overrideSource = overrideSource; + if (overrideSource == null) { + this.overrideLayer = null; + this.overrideSink = null; + this.readOnlyStore = baseStore; + } else { + this.overrideLayer = new OverrideLayer(); + this.overrideSink = new OverrideSinkImpl(overrideLayer, baseStore, flagChangeBroadcaster, + logger.subLogger(Loggers.DATA_SOURCE_LOGGER_NAME)); + this.readOnlyStore = new OverrideOverlayStore(baseStore, overrideLayer); + } } private static class FactoryWrapper implements FDv2DataSource.DataSourceFactory { @@ -187,13 +213,33 @@ static FDv2DataSystem create( FlagChangeNotifier flagChanged = new FlagChangedFacade(dataSourceUpdates); - return new FDv2DataSystem( + // The override source is built like any other component. Invalid configuration fails here, the + // same way an invalid data source or data store configuration fails. An offline client starts + // no data sources and no override source. + OverrideSource overrideSource = null; + if (dataSystemConfiguration.getOverrideSource() != null) { + if (config.offline) { + logger.warn("Offline mode is enabled; the override source is not started and no overrides are served"); + } else { + overrideSource = dataSystemConfiguration.getOverrideSource().build(clientContext); + } + } + + FDv2DataSystem system = new FDv2DataSystem( store, dataSource, dataSourceStatusProvider, dataStoreStatusProvider, - flagChanged + flagChanged, + flagChangeBroadcaster, + overrideSource, + logger ); + if (system.overrideLayer != null) { + // A LaunchDarkly change must also notify the flags whose overrides depend on it. + dataSourceUpdates.setExternalDependencies(system.overrideLayer); + } + return system; } @Override @@ -203,6 +249,27 @@ public ReadOnlyStore getStore() { @Override public Future start() { + if (overrideSource != null) { + // The source starts before the data source, so a source that loads synchronously has its + // overrides in place before the client begins evaluating. Its initial load is part of + // starting the client. A source that fails to start does not prevent the client from + // operating: the failure is logged, the source is closed, and the client runs without + // overrides. The SDK's own file-based source does not throw from start; it logs a failed + // load and retries it. A custom source could throw, and an application in an incident is + // better served by a client that runs without its overrides than by one that does not start. + try { + overrideSource.start(overrideSink); + } catch (RuntimeException e) { + logger.error("Unable to start the override source; the client runs without overrides: {}", + LogValues.exceptionSummary(e)); + logger.debug(LogValues.exceptionTrace(e)); + closeOverrideSource("Unable to close the override source after it failed to start: {}"); + // The source may have supplied a snapshot before it failed. Clearing the layer removes it, + // so the client runs without overrides as logged, and a late snapshot from the closed + // source is ignored. + overrideSink.closeAndClear(); + } + } return dataSource.start(); } @@ -231,12 +298,37 @@ public String getEnvironmentId() { return store.getEnvironmentId(); } + // A custom source can throw from close. That is logged and not thrown, so the rest of the client + // still shuts down, or still starts after a failed start. + private void closeOverrideSource(String failureMessage) { + OverrideSource source = overrideSource; + overrideSource = null; + try { + source.close(); + } catch (IOException | RuntimeException e) { + logger.warn(failureMessage, LogValues.exceptionSummary(e)); + logger.debug(LogValues.exceptionTrace(e)); + } + } + + @Override + public OverrideLayer getOverrideLayer() { + return overrideLayer; + } + @Override public void close() throws IOException { if (disposed) { return; } try { + if (overrideSource != null) { + // The sink stops accepting snapshots first, so a snapshot that the source supplies while it + // shuts down cannot update the layer or notify listeners through components that are being + // closed. + overrideSink.close(); + closeOverrideSource("Unable to close the override source: {}"); + } dataSource.close(); store.close(); } finally { diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FeatureFlagsState.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FeatureFlagsState.java index beafaaea..c4cc3476 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FeatureFlagsState.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FeatureFlagsState.java @@ -276,15 +276,19 @@ public Builder add( } Builder addFlag(DataModel.FeatureFlag flag, EvalResult eval) { + // A consumer of this state sends individual events according to the tracking fields. An + // override-affected evaluation produces no individual events, so the state turns them off + // for this flag. The flag, its value, its version, and its reason stay. + boolean overrideAffected = eval.isOverrideAffected(); return add( flag.getKey(), eval.getValue(), eval.isNoVariation() ? null : eval.getVariationIndex(), eval.getReason(), flag.getVersion(), - flag.isTrackEvents() || eval.isForceReasonTracking(), - eval.isForceReasonTracking(), - flag.getDebugEventsUntilDate(), + !overrideAffected && (flag.isTrackEvents() || eval.isForceReasonTracking()), + !overrideAffected && eval.isForceReasonTracking(), + overrideAffected ? null : flag.getDebugEventsUntilDate(), eval.getPrerequisiteEvalRecords().stream() .filter(record -> record.prereqOfFlag.getKey() == flag.getKey()) // only include top level prereqs .map(record -> record.flag.getKey()) // map from prereq record to prereq key diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/InputValidatingEvaluator.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/InputValidatingEvaluator.java index 8e7b61fd..59a8dde7 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/InputValidatingEvaluator.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/InputValidatingEvaluator.java @@ -13,7 +13,9 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.Collections; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import static com.launchdarkly.sdk.EvaluationDetail.NO_VARIATION; import static com.launchdarkly.sdk.server.DataModel.FEATURES; @@ -28,7 +30,11 @@ class InputValidatingEvaluator implements EvaluatorInterface { private final Evaluator evaluator; private final ReadOnlyStore store; + // Null when no override source is configured. The store already applies the layer. The layer is + // consulted directly only for the not-initialized short-circuit. + private final OverrideLayer overrideLayer; private final LDLogger logger; + private final AtomicBoolean allFlagsOverridesOnlyWarningLogged = new AtomicBoolean(false); // these are created at construction to avoid recreation during each evaluation private final EvaluationRecorder evaluationEventRecorderWithDetails; @@ -40,11 +46,14 @@ class InputValidatingEvaluator implements EvaluatorInterface { * Creates an {@link InputValidatingEvaluator} * * @param store will be used to get flag data + * @param overrideLayer the override layer, or null if no override source is configured * @param segmentStore will be used to get segment data * @param eventProcessor will be used to record events during evaluations as necessary * @param logger for logging messages and errors during evaluations */ - InputValidatingEvaluator(ReadOnlyStore store, BigSegmentStoreWrapper segmentStore, @Nonnull EventProcessor eventProcessor, LDLogger logger) { + InputValidatingEvaluator(ReadOnlyStore store, @Nullable OverrideLayer overrideLayer, BigSegmentStoreWrapper segmentStore, + @Nonnull EventProcessor eventProcessor, LDLogger logger) { + this.overrideLayer = overrideLayer; this.evaluator = new Evaluator(new Evaluator.Getters() { public DataModel.FeatureFlag getFlag(String key) { return InputValidatingEvaluator.getFlag(store, key); @@ -101,7 +110,9 @@ public EvalResultAndFlag evalAndFlag(String method, String flagKey, LDContext co */ EvalResultAndFlag evaluate(String flagKey, LDContext context, LDValue defaultValue, @Nullable LDValueType requireType, EvaluationRecorder recorder) { - if (!store.isInitialized()) { + // The override layer is consulted before the not-initialized short-circuit. A flag that the + // layer holds is served from it. Any other flag returns the not-ready default as before. + if (!store.isInitialized() && !hasOverrideFor(flagKey)) { logger.warn("Evaluation called before client initialized for feature flag \"{}\"; data store unavailable, returning default value", flagKey); recorder.recordEvaluationUnknownFlagError(flagKey, context, defaultValue, ErrorKind.CLIENT_NOT_READY); return new EvalResultAndFlag(EvalResult.error(ErrorKind.CLIENT_NOT_READY, defaultValue), null); @@ -134,8 +145,13 @@ EvalResultAndFlag evaluate(String flagKey, LDContext context, LDValue defaultVal !value.isNull() && value.getType() != requireType) { logger.error("Feature flag \"{}\"; evaluation expected result as {}, but got {}", flagKey, defaultValue.getType(), value.getType()); - recorder.recordEvaluationError(featureFlag, context, defaultValue, ErrorKind.WRONG_TYPE); - return new EvalResultAndFlag(EvalResult.error(ErrorKind.WRONG_TYPE, defaultValue), featureFlag); + recorder.recordEvaluationError(featureFlag, context, defaultValue, ErrorKind.WRONG_TYPE, + result.isOverrideAffected()); + // The type mismatch replaces the reason. The evaluation read the same definitions, so the + // new reason keeps the override-affected marking. + return new EvalResultAndFlag( + EvalResult.error(ErrorKind.WRONG_TYPE, defaultValue).withOverrideAffected(result.isOverrideAffected()), + featureFlag); } } @@ -146,21 +162,40 @@ EvalResultAndFlag evaluate(String flagKey, LDContext context, LDValue defaultVal logger.error("Encountered exception while evaluating feature flag \"{}\": {}", flagKey, LogValues.exceptionSummary(e)); logger.debug("{}", LogValues.exceptionTrace(e)); + // The flag's definition was read before the failure, so its own marker decides the marking of + // the error event and of the error result. + boolean overrideAffected = featureFlag != null && featureFlag.isOverride(); if (featureFlag == null) { recorder.recordEvaluationUnknownFlagError(flagKey, context, defaultValue, ErrorKind.EXCEPTION); } else { - recorder.recordEvaluationError(featureFlag, context, defaultValue, ErrorKind.EXCEPTION); + recorder.recordEvaluationError(featureFlag, context, defaultValue, ErrorKind.EXCEPTION, overrideAffected); } - return new EvalResultAndFlag(EvalResult.of(defaultValue, NO_VARIATION, EvaluationReason.exception(e)), null); + return new EvalResultAndFlag( + EvalResult.of(defaultValue, NO_VARIATION, EvaluationReason.exception(e)).withOverrideAffected(overrideAffected), + null); } } public FeatureFlagsState allFlagsState(LDContext context, FlagsStateOption... options) { FeatureFlagsState.Builder builder = FeatureFlagsState.builder(options); + DataStoreTypes.KeyedItems overrideFlags = null; if (!store.isInitialized()) { - logger.warn("allFlagsState() was called before client initialized; data store unavailable, returning no data"); - return builder.valid(false).build(); + // No LaunchDarkly data is available. When the override layer holds flags, the state contains + // only those, read from the layer itself: a store that is not initialized can still hold + // LaunchDarkly data, which the state must not include. Without override flags the state is + // invalid, as it is without an override source. + Map layerFlags = + overrideLayer == null ? Collections.emptyMap() : overrideLayer.all(FEATURES); + if (layerFlags.isEmpty()) { + logger.warn("allFlagsState() was called before client initialized; data store unavailable, returning no data"); + return builder.valid(false).build(); + } + if (allFlagsOverridesOnlyWarningLogged.compareAndSet(false, true)) { + logger.warn("allFlagsState() was called before client initialized; returning only flags from the override layer." + + " This message is logged once."); + } + overrideFlags = new DataStoreTypes.KeyedItems<>(layerFlags.entrySet()); } if (context == null) { @@ -174,12 +209,27 @@ public FeatureFlagsState allFlagsState(LDContext context, FlagsStateOption... op boolean clientSideOnly = FlagsStateOption.hasOption(options, FlagsStateOption.CLIENT_SIDE_ONLY); DataStoreTypes.KeyedItems flags; - try { - flags = store.getAll(FEATURES); - } catch (Exception e) { - logger.error("Exception from data store when evaluating all flags: {}", LogValues.exceptionSummary(e)); - logger.debug(e.toString(), LogValues.exceptionTrace(e)); - return builder.valid(false).build(); + boolean valid = true; + if (overrideFlags != null) { + flags = overrideFlags; + } else { + try { + flags = store.getAll(FEATURES); + } catch (Exception e) { + logger.error("Exception from data store when evaluating all flags: {}", LogValues.exceptionSummary(e)); + logger.debug(e.toString(), LogValues.exceptionTrace(e)); + // Overrides only add to the state. Without them the state is invalid and empty here, so it + // holds the override layer's flags and stays invalid. A client-side SDK bootstrapped from the + // state uses its flags whatever $valid says (js-client-sdk 3.x and the 4.x browser SDK only + // log a warning), so the overridden values still reach it. + Map layerFlags = + overrideLayer == null ? Collections.emptyMap() : overrideLayer.all(FEATURES); + if (layerFlags.isEmpty()) { + return builder.valid(false).build(); + } + flags = new DataStoreTypes.KeyedItems<>(layerFlags.entrySet()); + valid = false; + } } for (Map.Entry entry : flags.getItems()) { @@ -199,10 +249,13 @@ public FeatureFlagsState allFlagsState(LDContext context, FlagsStateOption... op logger.error("Exception caught for feature flag \"{}\" when evaluating all flags: {}", flag.getKey(), LogValues.exceptionSummary(e)); logger.debug(e.toString(), LogValues.exceptionTrace(e)); - builder.addFlag(flag, EvalResult.of(LDValue.ofNull(), NO_VARIATION, EvaluationReason.exception(e))); + // The flag's definition was read before the failure, so its own marker decides the marking, + // and with it the state's event tracking fields for the flag. + builder.addFlag(flag, EvalResult.of(LDValue.ofNull(), NO_VARIATION, EvaluationReason.exception(e)) + .withOverrideAffected(flag.isOverride())); } } - return builder.build(); + return builder.valid(valid).build(); } private static DataModel.FeatureFlag getFlag(ReadOnlyStore store, String key) { @@ -210,6 +263,14 @@ private static DataModel.FeatureFlag getFlag(ReadOnlyStore store, String key) { return item == null ? null : (DataModel.FeatureFlag) item.getItem(); } + private boolean hasOverrideFor(String flagKey) { + if (overrideLayer == null) { + return false; + } + DataStoreTypes.ItemDescriptor item = overrideLayer.get(FEATURES, flagKey); + return item != null && item.getItem() != null; + } + private static DataModel.Segment getSegment(ReadOnlyStore store, String key) { DataStoreTypes.ItemDescriptor item = store.get(SEGMENTS, key); return item == null ? null : (DataModel.Segment) item.getItem(); @@ -225,6 +286,8 @@ private static DataModel.Segment getSegment(ReadOnlyStore store, String key) { */ private static EvaluationRecorder makeEvaluationRecorder(EventProcessor processor, boolean withReasons) { return new EvaluationRecorder() { + // The marking passed to the event processor is the evaluation result's own record of the + // definitions it read. Event generation does not read the evaluation reason. @Override public void recordEvaluation(FeatureFlag flag, LDContext context, EvalResult result, LDValue defaultValue) { processor.recordEvaluationEvent( @@ -239,10 +302,13 @@ public void recordEvaluation(FeatureFlag flag, LDContext context, EvalResult res flag.isTrackEvents() || result.isForceReasonTracking(), flag.getDebugEventsUntilDate(), flag.isExcludeFromSummaries(), - flag.getSamplingRatio() + flag.getSamplingRatio(), + result.isOverrideAffected() ); } + // A prerequisite record carries the prerequisite's own marking, which its own subtree of + // reads set. It is not the marking of the evaluation that requested it. @Override public void recordPrerequisiteEvaluation(FeatureFlag flag, FeatureFlag prereqOfFlag, LDContext context, EvalResult result) { processor.recordEvaluationEvent( @@ -257,25 +323,28 @@ public void recordPrerequisiteEvaluation(FeatureFlag flag, FeatureFlag prereqOfF flag.isTrackEvents() || result.isForceReasonTracking(), flag.getDebugEventsUntilDate(), flag.isExcludeFromSummaries(), - flag.getSamplingRatio() + flag.getSamplingRatio(), + result.isOverrideAffected() ); } @Override - public void recordEvaluationError(FeatureFlag flag, LDContext context, LDValue defaultValue, ErrorKind errorKind) { + public void recordEvaluationError(FeatureFlag flag, LDContext context, LDValue defaultValue, ErrorKind errorKind, + boolean overrideAffected) { processor.recordEvaluationEvent( context, flag.getKey(), flag.getVersion(), NO_VARIATION, defaultValue, - withReasons ? EvaluationReason.error(errorKind) : null, + withReasons ? EvaluationReason.error(errorKind).withOverrideAffected(overrideAffected) : null, defaultValue, null, flag.isTrackEvents(), flag.getDebugEventsUntilDate(), flag.isExcludeFromSummaries(), - flag.getSamplingRatio() + flag.getSamplingRatio(), + overrideAffected ); } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/LDClient.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/LDClient.java index 18f314ca..e0c899d8 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/LDClient.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/LDClient.java @@ -209,7 +209,8 @@ public LDClient(String sdkKey, LDConfig config) { this.dataSystem = FDv1DataSystem.create(baseLogger, config, context, context.getLogging()); } - EvaluatorInterface evaluator = new InputValidatingEvaluator(this.dataSystem.getStore(), bigSegmentStoreWrapper, eventProcessor, evaluationLogger); + EvaluatorInterface evaluator = new InputValidatingEvaluator(this.dataSystem.getStore(), + this.dataSystem.getOverrideLayer(), bigSegmentStoreWrapper, eventProcessor, evaluationLogger); // build environment metadata for plugins SdkMetadata sdkMetadata; @@ -428,6 +429,17 @@ public MigrationVariation migrationVariation(String key, LDContext context, Migr @Override public boolean isFlagKnown(String featureKey) { + // An override makes a flag known whether or not the client is initialized, the same as + // evaluation serves an override before the not-initialized short circuit (spec 1.5.3). A + // caller that gates on isFlagKnown must not skip an override during an outage. + OverrideLayer overrideLayer = dataSystem.getOverrideLayer(); + if (overrideLayer != null) { + ItemDescriptor overridden = overrideLayer.get(DataModel.FEATURES, featureKey); + if (overridden != null && overridden.getItem() != null) { + return true; + } + } + ReadOnlyStore store = dataSystem.getStore(); if (!isInitialized()) { diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/MigrationStageEnforcingEvaluator.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/MigrationStageEnforcingEvaluator.java index bdf2be3f..806a8bb2 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/MigrationStageEnforcingEvaluator.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/MigrationStageEnforcingEvaluator.java @@ -28,7 +28,10 @@ public EvalResultAndFlag evalAndFlag(String method, String flagKey, LDContext co String resStageString = resDetail.getValue(); if (!MigrationStage.isStage(resStageString)) { logger.error("Unrecognized MigrationState for \"{}\"; returning default value.", flagKey); - return new EvalResultAndFlag(EvalResult.error(EvaluationReason.ErrorKind.WRONG_TYPE, defaultValue), res.getFlag()); + // The unrecognized stage replaces the result. The evaluation read the same definitions, so + // the new result keeps the override-affected marking. + return new EvalResultAndFlag(EvalResult.error(EvaluationReason.ErrorKind.WRONG_TYPE, defaultValue) + .withOverrideAffected(res.getResult().isOverrideAffected()), res.getFlag()); } return res; diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/NoOpEventProcessor.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/NoOpEventProcessor.java index 60684967..37b08ce4 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/NoOpEventProcessor.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/NoOpEventProcessor.java @@ -20,6 +20,14 @@ public void recordEvaluationEvent(LDContext context, String flagKey, int flagVer // no-op } + @Override + public void recordEvaluationEvent(LDContext context, String flagKey, int flagVersion, int variation, LDValue value, + EvaluationReason reason, LDValue defaultValue, String prerequisiteOfFlagKey, + boolean requireFullEvent, Long debugEventsUntilDate, boolean excludeFromSummaries, + Long samplingRatio, boolean overrideAffected) { + // no-op + } + @Override public void recordIdentifyEvent(LDContext context) { // no-op diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/OverrideLayer.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/OverrideLayer.java new file mode 100644 index 00000000..ce1f55ea --- /dev/null +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/OverrideLayer.java @@ -0,0 +1,152 @@ +package com.launchdarkly.sdk.server; + +import com.google.common.collect.ImmutableMap; +import com.launchdarkly.sdk.server.DataModel.FeatureFlag; +import com.launchdarkly.sdk.server.DataModel.Segment; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.DataKind; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.KeyedItems; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * The override layer: a thread-safe store of override entries that an override source replaces + * wholesale on each update. Entries take precedence over LaunchDarkly data at the store read + * boundary; see {@link OverrideOverlayStore}. + *

+ * Each entry is a marked shallow copy of the entity that the source supplied. The copy shares its + * nested collections with the source's entity, and the layer never writes to them. The source's + * entity is never marked, so a source may retain the entities it supplied and supply them again. + *

+ * This class is package-private and should not be used by application code. + */ +final class OverrideLayer implements DataModelDependencies.ExternalDependencies { + /** + * The previous and current contents of the layer after a replacement. The maps must not be + * modified. + */ + static final class Replacement { + final ImmutableMap> previous; + final ImmutableMap> current; + + Replacement( + ImmutableMap> previous, + ImmutableMap> current + ) { + this.previous = previous; + this.current = current; + } + } + + // The contents are an immutable map that is swapped on each update, so the layer holds exactly + // one snapshot at any instant and readers never take a lock. + private volatile ImmutableMap> contents = ImmutableMap.of(); + // A single volatile read decides the common case of a configured but unpopulated layer. + private volatile boolean nonEmpty = false; + private final Object writeLock = new Object(); + // The dependency edges of the layer's own entries: which entries reference which flags and + // segments. Rebuilt with the contents on every replacement and swapped with them. + private volatile DataModelDependencies.DependencyTracker dependencies = new DataModelDependencies.DependencyTracker(); + + /** + * Atomically replaces the entire layer contents. A null or empty collection clears the layer. + * + * @param data the complete set of entries, grouped by data kind + * @return the previous and the new contents + */ + Replacement setAll(Iterable>> data) { + Map> collected = new HashMap<>(); + DataModelDependencies.DependencyTracker tracker = new DataModelDependencies.DependencyTracker(); + int count = 0; + if (data != null) { + for (Map.Entry> kindEntry : data) { + Map items = collected.computeIfAbsent(kindEntry.getKey(), k -> new HashMap<>()); + Iterable> kindItems = kindEntry.getValue() == null + ? null : kindEntry.getValue().getItems(); + if (kindItems == null) { + continue; + } + for (Map.Entry item : kindItems) { + ItemDescriptor copied = markedCopy(item.getValue()); + items.put(item.getKey(), copied); + tracker.updateDependenciesFrom(kindEntry.getKey(), item.getKey(), copied); + count++; + } + } + } + ImmutableMap.Builder> builder = ImmutableMap.builder(); + for (Map.Entry> e : collected.entrySet()) { + builder.put(e.getKey(), ImmutableMap.copyOf(e.getValue())); + } + ImmutableMap> replacement = builder.build(); + + synchronized (writeLock) { + ImmutableMap> previous = contents; + contents = replacement; + dependencies = tracker; + nonEmpty = count != 0; + return new Replacement(previous, replacement); + } + } + + /** + * Adds the given item to the set together with every item whose override entry depends on it, + * directly or through other override entries. A LaunchDarkly update uses this so that a change to + * a LaunchDarkly segment or prerequisite notifies the overridden flags that reference it, which + * the data system's own dependency tracking cannot see. + */ + @Override + public void addAffectedItems(Set itemsOut, DataModelDependencies.KindAndKey item) { + dependencies.addAffectedItems(itemsOut, item); + } + + /** + * Returns the override entry for a key, or null if the layer has none. + * + * @param kind the data kind + * @param key the item key + * @return the marked entry or null + */ + ItemDescriptor get(DataKind kind, String key) { + if (!nonEmpty) { + return null; + } + ImmutableMap items = contents.get(kind); + return items == null ? null : items.get(key); + } + + /** + * Returns the entries of a kind. The map must not be modified. + * + * @param kind the data kind + * @return the entries, possibly empty + */ + ImmutableMap all(DataKind kind) { + ImmutableMap items = contents.get(kind); + return items == null ? ImmutableMap.of() : items; + } + + /** + * Reports whether the layer contains no entries. + * + * @return true if empty + */ + boolean isEmpty() { + return !nonEmpty; + } + + // Returns a copy of the descriptor whose item carries the override marker. An item of another + // type, or a deleted item placeholder, is returned as is. + static ItemDescriptor markedCopy(ItemDescriptor item) { + Object entity = item.getItem(); + if (entity instanceof FeatureFlag) { + return new ItemDescriptor(item.getVersion(), ((FeatureFlag) entity).markedAsOverride()); + } + if (entity instanceof Segment) { + return new ItemDescriptor(item.getVersion(), ((Segment) entity).markedAsOverride()); + } + return item; + } +} diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/OverrideOverlayStore.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/OverrideOverlayStore.java new file mode 100644 index 00000000..5901145e --- /dev/null +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/OverrideOverlayStore.java @@ -0,0 +1,84 @@ +package com.launchdarkly.sdk.server; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.DataKind; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.KeyedItems; + +import java.util.AbstractMap; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * Merges an {@link OverrideLayer} over a base store. A read for a key returns the override entry + * when one exists, and the base entry otherwise. The overlay sits at the store read boundary, so + * targeting rules, prerequisites, and segment matches behave identically for overridden and + * ordinary data: they are the same reads through the same boundary. + *

+ * This class is package-private and should not be used by application code. + */ +final class OverrideOverlayStore implements ReadOnlyStore { + private final ReadOnlyStore base; + private final OverrideLayer layer; + + OverrideOverlayStore(ReadOnlyStore base, OverrideLayer layer) { + this.base = base; + this.layer = layer; + } + + /** + * Returns the override entry for the key if one exists, and otherwise delegates to the base + * store. This works even when the base store is uninitialized, because an uninitialized base + * reports not found rather than failing. + */ + @Override + public ItemDescriptor get(DataKind kind, String key) { + ItemDescriptor override = layer.get(kind, key); + if (override != null) { + return override; + } + return base.get(kind, key); + } + + /** + * Returns the union of the base store's items and the layer's items. The override entry wins for + * any key present in both, including keys that the base holds as deleted-item placeholders. A + * failure of the base store propagates, as it does without the overlay. + */ + @Override + public KeyedItems getAll(DataKind kind) { + KeyedItems fromBase = base.getAll(kind); + ImmutableMap overrides = layer.all(kind); + if (overrides.isEmpty()) { + return fromBase; + } + Iterable> baseItems = + fromBase == null || fromBase.getItems() == null ? Collections.emptyList() : fromBase.getItems(); + + ImmutableList.Builder> result = ImmutableList.builder(); + Set seen = new HashSet<>(); + for (Map.Entry item : baseItems) { + ItemDescriptor override = overrides.get(item.getKey()); + result.add(override == null ? item : new AbstractMap.SimpleEntry<>(item.getKey(), override)); + seen.add(item.getKey()); + } + for (Map.Entry override : overrides.entrySet()) { + if (!seen.contains(override.getKey())) { + result.add(override); + } + } + return new KeyedItems<>(result.build()); + } + + /** + * Delegates to the base store. The override layer never affects initialization status or data + * availability. + */ + @Override + public boolean isInitialized() { + return base.isInitialized(); + } +} diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/OverrideSinkImpl.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/OverrideSinkImpl.java new file mode 100644 index 00000000..280eed07 --- /dev/null +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/OverrideSinkImpl.java @@ -0,0 +1,223 @@ +package com.launchdarkly.sdk.server; + +import com.google.common.collect.ImmutableMap; +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.sdk.server.DataModelDependencies.DependencyTracker; +import com.launchdarkly.sdk.server.DataModelDependencies.KindAndKey; +import com.launchdarkly.sdk.server.interfaces.FlagChangeEvent; +import com.launchdarkly.sdk.server.interfaces.FlagChangeListener; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.DataKind; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.KeyedItems; +import com.launchdarkly.sdk.server.subsystems.OverrideSink; + +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import static com.launchdarkly.sdk.server.DataModel.FEATURES; +import static com.launchdarkly.sdk.server.DataModel.SEGMENTS; + +/** + * Applies override layer replacements supplied by an override source, and notifies flag change + * listeners of the flags affected by each replacement. + *

+ * This class is package-private and should not be used by application code. + */ +final class OverrideSinkImpl implements OverrideSink { + private static final DataKind[] DIFF_KINDS = new DataKind[] { FEATURES, SEGMENTS }; + + private final OverrideLayer layer; + private final ReadOnlyStore base; + private final EventBroadcasterImpl flagChangeBroadcaster; + private final LDLogger logger; + private boolean closed; // guarded by this + + /** + * Creates a sink that writes to the given layer. + * + * @param layer the override layer + * @param base the raw store that holds LaunchDarkly data, without the overlay. Merged-view + * snapshots for change computation are built from it plus the layer. + * @param flagChangeBroadcaster the client's flag change broadcaster + * @param logger the logger + */ + OverrideSinkImpl( + OverrideLayer layer, + ReadOnlyStore base, + EventBroadcasterImpl flagChangeBroadcaster, + LDLogger logger + ) { + this.layer = layer; + this.base = base; + this.flagChangeBroadcaster = flagChangeBroadcaster; + this.logger = logger; + } + + /** + * Atomically replaces the entire override layer, then notifies listeners of every flag whose + * merged-view evaluation may have changed. Calls are serialized, so overlapping updates from a + * source cannot interleave. + */ + @Override + public synchronized void setOverrides(Iterable>> data) { + if (closed) { + logger.debug("Ignoring an override snapshot supplied after the override source was closed"); + return; + } + apply(data); + } + + /** + * Stops accepting snapshots. Used when the client is closed: a snapshot that the source supplies + * while it shuts down is ignored. The layer keeps its contents. + */ + synchronized void close() { + closed = true; + } + + /** + * Stops accepting snapshots and clears the layer. Used when the source failed to start: a snapshot + * the source supplied before failing is removed, so the client runs without overrides, and a late + * snapshot from the closed source is ignored. + */ + synchronized void closeAndClear() { + closed = true; + apply(Collections.emptyList()); + } + + // Called while holding the lock on this. + private void apply(Iterable>> data) { + // Computing affected flags requires snapshots of the merged view before and after the + // replacement. Skip all of that work when nothing is listening. + if (!flagChangeBroadcaster.hasListeners()) { + layer.setAll(data); + return; + } + + OverrideLayer.Replacement replacement = layer.setAll(data); + Map> oldMerged = snapshotMergedView(replacement.previous); + Map> newMerged = snapshotMergedView(replacement.current); + + Set affected = computeAffectedFlags(replacement.previous, replacement.current, oldMerged, newMerged); + if (!affected.isEmpty()) { + logger.debug("Override update affected {} flag(s)", affected.size()); + } + for (String key : affected) { + flagChangeBroadcaster.broadcast(new FlagChangeEvent(key)); + } + } + + // Returns the keys of all flags whose merged-view evaluation may have changed when the layer was + // replaced. The result includes the flags whose override entries were added, removed, or changed. + // Dependency fan-out adds every flag that depends, directly or transitively, on any added, + // removed, or changed entry of either kind. + static Set computeAffectedFlags( + ImmutableMap> oldOverrides, + ImmutableMap> newOverrides, + Map> oldMerged, + Map> newMerged + ) { + Set seeds = diffOverrides(oldOverrides, newOverrides); + Set flagKeys = new LinkedHashSet<>(); + if (seeds.isEmpty()) { + return flagKeys; + } + + // Dependency edges are computed over both the old and the new merged views, because a + // replacement can rewire dependencies. For example, removing a flag override restores the + // prerequisite edges of the LaunchDarkly definition. Flags that depended on the override's + // references exist as dependents only in the old view. + DependencyTracker oldTracker = trackerFromView(oldMerged); + DependencyTracker newTracker = trackerFromView(newMerged); + Set affected = new LinkedHashSet<>(); + for (KindAndKey seed : seeds) { + oldTracker.addAffectedItems(affected, seed); + newTracker.addAffectedItems(affected, seed); + } + for (KindAndKey item : affected) { + if (item.kind == FEATURES) { + flagKeys.add(item.key); + } + } + return flagKeys; + } + + // Returns each key whose override entry differs between the two layer snapshots. An added or + // removed entry is always a change, even when its content matches the underlying LaunchDarkly + // data: the override marker alone changes the served entry. Entries present in both snapshots + // are compared by version and serialized form, because the layer is rebuilt wholesale on every + // update and identity comparison would report every retained entry as changed. + private static Set diffOverrides( + ImmutableMap> oldOverrides, + ImmutableMap> newOverrides + ) { + Set seeds = new LinkedHashSet<>(); + for (DataKind kind : DIFF_KINDS) { + Map oldItems = oldOverrides.getOrDefault(kind, ImmutableMap.of()); + Map newItems = newOverrides.getOrDefault(kind, ImmutableMap.of()); + for (Map.Entry e : oldItems.entrySet()) { + ItemDescriptor newItem = newItems.get(e.getKey()); + if (newItem == null || !itemsEqual(kind, e.getValue(), newItem)) { + seeds.add(new KindAndKey(kind, e.getKey())); + } + } + for (String key : newItems.keySet()) { + if (!oldItems.containsKey(key)) { + seeds.add(new KindAndKey(kind, key)); + } + } + } + return seeds; + } + + private static boolean itemsEqual(DataKind kind, ItemDescriptor a, ItemDescriptor b) { + if (a.getVersion() != b.getVersion()) { + return false; + } + return Objects.equals(kind.serialize(a), kind.serialize(b)); + } + + // Captures the data visible at the store read boundary: base data with the given override entries + // overlaid. A base read failure for a kind yields just the overrides for that kind. This degrades + // the dependency fan-out but never loses the directly changed keys. + private Map> snapshotMergedView( + ImmutableMap> overrides + ) { + Map> view = new HashMap<>(); + for (DataKind kind : DIFF_KINDS) { + Map items = new HashMap<>(); + try { + KeyedItems baseItems = base.getAll(kind); + if (baseItems != null && baseItems.getItems() != null) { + for (Map.Entry item : baseItems.getItems()) { + items.put(item.getKey(), item.getValue()); + } + } + } catch (RuntimeException e) { + logger.debug("Unable to read {} from the data store while computing override changes: {}", + kind.getName(), e.toString()); + } + items.putAll(overrides.getOrDefault(kind, ImmutableMap.of())); + view.put(kind, items); + } + return view; + } + + private static DependencyTracker trackerFromView(Map> view) { + DependencyTracker tracker = new DependencyTracker(); + for (DataKind kind : DIFF_KINDS) { + Map items = view.get(kind); + if (items == null) { + continue; + } + for (Map.Entry e : items.entrySet()) { + tracker.updateDependenciesFrom(kind, e.getKey(), e.getValue()); + } + } + return tracker; + } +} diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/DataSystemBuilder.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/DataSystemBuilder.java index 73d1f3b3..fdaa3093 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/DataSystemBuilder.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/DataSystemBuilder.java @@ -19,6 +19,7 @@ public final class DataSystemBuilder { private ComponentConfigurer fDv1FallbackSynchronizer; private ComponentConfigurer persistentStore; private DataSystemConfiguration.DataStoreMode persistentDataStoreMode; + private ComponentConfigurer overrideSource; /** * Add one or more initializers to the builder. @@ -108,13 +109,50 @@ public DataSystemBuilder persistentStore(ComponentConfigurer persiste return this; } + /** + * Configures an override source. Flag overrides are currently experimental and subject to change. + *

+ * The source supplies flag and segment definitions that take precedence over data received from + * LaunchDarkly on a per-key basis. Overrides let an operator force one or more flags to a known + * state on a running client, whether or not the client can reach LaunchDarkly. Flags not present + * in the override data are unaffected. + *

+ *

+ * The override source is not a data source. It has no effect on the client's initialization + * status or data source status. Configuring it changes nothing until the source supplies an + * override. At most one override source can be configured; a later call replaces the earlier one. + *

+ *

+ * When the client is offline, the source is not built or started: no overrides are served, and a + * configuration error in the source is not reported. The SDK logs a warning instead. + *

+ *

+ * Example: + *

+ * 

+   *     LDConfig config = new LDConfig.Builder()
+   *       .dataSystem(Components.dataSystem().defaultMode()
+   *         .overrides(FileOverrides.source().filePaths("/etc/launchdarkly/overrides.json")))
+   *       .build();
+   *
+ * + * @param overrideSource the override source configuration, or null for none + * @return a reference to the builder + * @see FileOverrides + * @since 7.18.0 + */ + public DataSystemBuilder overrides(ComponentConfigurer overrideSource) { + this.overrideSource = overrideSource; + return this; + } + /** * Build the data system configuration. *

* This method is internal and should not be called by application code. * This function should remain internal. *

- * + * * @return the data system configuration */ public DataSystemConfiguration build() { @@ -123,7 +161,8 @@ public DataSystemConfiguration build() { ImmutableList.copyOf(synchronizers), fDv1FallbackSynchronizer, persistentStore, - persistentDataStoreMode); + persistentDataStoreMode, + overrideSource); } } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileDataPoller.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileDataPoller.java index 954e22e6..65100cb8 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileDataPoller.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileDataPoller.java @@ -104,7 +104,14 @@ static List observeAll(List paths) { return states; } - private static FileState observe(Path path) { + /** + * Observes one file. The attributes are read through any symbolic link, so they describe the file + * that the link points to. + * + * @param path the file + * @return its state, or {@link FileState#ABSENT} if its attributes cannot be read + */ + static FileState observe(Path path) { try { BasicFileAttributes attributes = Files.readAttributes(path, BasicFileAttributes.class); return new FileState(true, attributes.lastModifiedTime().toMillis(), attributes.size()); diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileDataReloader.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileDataReloader.java index 97a73c07..e844d1ca 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileDataReloader.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileDataReloader.java @@ -6,8 +6,12 @@ import com.launchdarkly.sdk.server.integrations.OverrideFileLoader.LoadResult; import java.io.Closeable; +import java.nio.file.Path; import java.time.Duration; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.ThreadFactory; @@ -23,6 +27,10 @@ * and returns the merged result, or throws {@link FileDataException} when a file cannot be read or * parsed or when the files cannot be merged. *

+ * A file whose content is in effect and is found empty may be in the middle of an in-place rewrite. + * The loader can hold such a file back with {@link EmptiedFileException}: the reloader keeps the + * current data, reports no failure, and reads the files again when the exception says. + *

* The file data source does not use this class. It keeps its own reload behavior. */ final class FileDataReloader implements Closeable { @@ -43,7 +51,55 @@ final class FileDataReloader implements Closeable { * Performs one full load of all configured files. */ interface Loader { - LoadResult load() throws FileDataException; + /** + * Reads every configured file and merges the entries. + * + * @param retryDelay the reloader's retry delay, or zero if it does not retry. A loader may hold + * an empty file back for a time based on it, because a retry reads the files again. With zero, + * nothing would read them again, so it must not hold any back. + * @return the merged result + * @throws FileDataException if a file cannot be read or parsed, or the files cannot be merged + * @throws EmptiedFileException if a file was found empty and is held back + */ + LoadResult load(Duration retryDelay) throws FileDataException, EmptiedFileException; + + /** + * Called with the result of a load that is now the data in effect: it was applied, or it was + * skipped because it is unchanged. It is called with the reload lock held. + * + * @param result the result + */ + void accepted(LoadResult result); + } + + /** + * Thrown by a {@link Loader} that holds back files it found empty. A file may be in the middle of an + * in-place rewrite: a writer that truncates the file and then writes the new content. + */ + @SuppressWarnings("serial") + static final class EmptiedFileException extends Exception { + private final List paths; + private final long retryAfterNanos; + + EmptiedFileException(List paths, long retryAfterNanos) { + super("files " + paths + " are empty"); + this.paths = Collections.unmodifiableList(new ArrayList<>(paths)); + this.retryAfterNanos = retryAfterNanos; + } + + /** + * The files held back, in configuration order. + */ + List getPaths() { + return paths; + } + + /** + * How long to wait before loading again, in nanoseconds. + */ + long getRetryAfterNanos() { + return retryAfterNanos; + } } /** @@ -66,7 +122,8 @@ interface Handler { * Called when a reload fails, including when {@link #apply(LoadResult)} throws, once per * distinct failure. With automatic retries, repeats of an identical failure do not call this * again. A success re-arms it. The reloader logs failures itself, so implementations only need - * to update their own state. + * to update their own state. A load that defers an empty file and reads it again later is not a + * failure and does not call this. * * @param e the failure */ @@ -78,18 +135,26 @@ interface Handler { private final LDLogger logger; private final long debounceDelayMillis; private final long retryDelayMillis; + // What reload() returns when no retry is needed. + private static final long NO_RETRY = -1; private final boolean skipUnchanged; // Serializes the load work between reloadNow and the worker thread. private final Object reloadLock = new Object(); private byte[] lastGoodHash = null; private String lastErrorMessage = null; + // True from a load that found an emptied file until the next load starts. Written inside the + // reload lock. It only chooses the log message of the retry that such a load arms. + private volatile boolean readAgainPending = false; // Guards the worker and the two timers. private final Object timerLock = new Object(); private ScheduledThreadPoolExecutor worker = null; private ScheduledFuture debounceFuture = null; private ScheduledFuture retryFuture = null; + // The delay of the latest request to arm a retry, in nanoseconds, or -1 before the first. Visible for + // tests through getLastRequestedRetryNanos. + private volatile long lastRequestedRetryNanos = -1; private final AtomicBoolean closed = new AtomicBoolean(false); @@ -97,16 +162,17 @@ interface Handler { * Creates a reloader. The caller performs the initial load with {@link #reloadNow()}, routes * change signals to {@link #trigger()}, and calls {@link #close()} when finished. *

- * The worker thread starts on the first {@link #trigger()} call or the first failed load that - * arms a retry, not here. Construction alone does not start a thread. + * The worker thread starts on the first {@link #trigger()} call or the first load that arms a + * retry, not here. Construction alone does not start a thread. * * @param loader performs each full load * @param handler receives results and failures * @param logger receives log output about reloads and failures * @param debounceDelay how long to wait after a trigger for further triggers to settle before * reloading. Zero or negative reloads on every trigger without waiting. - * @param retryDelay how long to wait after a failed reload before retrying without a trigger. - * Zero or negative disables the automatic retry. + * @param retryDelay how long to wait after a failed reload before retrying without a trigger. A + * load that holds back an empty file says itself when to load again. Zero or negative disables + * the automatic retry, and the loader then parses an empty file as it is. * @param skipUnchanged if true, a successful load whose raw file contents are identical to the * last applied contents does not call the apply callback */ @@ -131,8 +197,9 @@ interface Handler { * initial load. A failure here arms the same automatic retry as a failed triggered reload. */ void reloadNow() { - if (!reload() && retryDelayMillis > 0) { - armRetry(); + long retryAfterNanos = reload(); + if (retryAfterNanos != NO_RETRY) { + armRetry(retryAfterNanos); } } @@ -181,6 +248,16 @@ public void close() { } } + /** + * Returns the delay of the latest request to arm a retry, whether or not it armed one. Visible for + * tests. + * + * @return the delay in nanoseconds, or -1 if no retry was requested + */ + long getLastRequestedRetryNanos() { + return lastRequestedRetryNanos; + } + /** * Reports whether the worker thread has been started. Visible for tests. * @@ -228,7 +305,8 @@ private ScheduledThreadPoolExecutor ensureWorker() { return worker; } - private void armRetry() { + private void armRetry(long delayNanos) { + lastRequestedRetryNanos = delayNanos; synchronized (timerLock) { if (closed.get()) { return; @@ -237,7 +315,7 @@ private void armRetry() { if (retryFuture != null && !retryFuture.isDone()) { return; } - retryFuture = ensureWorker().schedule(() -> runReload(true), retryDelayMillis, TimeUnit.MILLISECONDS); + retryFuture = ensureWorker().schedule(() -> runReload(true), delayNanos, TimeUnit.NANOSECONDS); } } @@ -245,7 +323,8 @@ private void armRetry() { private void runReload(boolean isRetry) { try { if (isRetry) { - logger.debug("Retrying flag data load after earlier failure"); + logger.debug(readAgainPending ? "Reading flag data again after a file was found empty" + : "Retrying flag data load after earlier failure"); } else { logger.info("Reloading flag data after detecting a change"); } @@ -257,8 +336,9 @@ private void runReload(boolean isRetry) { retryFuture = null; } } - if (!reload() && retryDelayMillis > 0) { - armRetry(); + long retryAfterNanos = reload(); + if (retryAfterNanos != NO_RETRY) { + armRetry(retryAfterNanos); } } catch (RuntimeException e) { // The executor would otherwise swallow the exception and the worker would go quiet. @@ -267,27 +347,31 @@ private void runReload(boolean isRetry) { } } - // Performs one full load of all configured files. It reports whether the load succeeded, which - // decides whether a retry gets armed. A skipped no-op application counts as success. The whole - // set is re-read on every reload because entries are combined across files in order, so a - // change to one file can alter which file wins for a key. - private boolean reload() { + // Performs one full load of all configured files. It returns when to load again: NO_RETRY after a + // success, including a skipped no-op application, the retry delay after a failure, or the time that + // a load that holds back an empty file asks for. The whole set is re-read on every reload because + // entries are combined across files in order, so a change to one file can alter which file wins + // for a key. + private long reload() { synchronized (reloadLock) { // A trigger already queued when close was called can still reach here. if (closed.get()) { - return true; + return NO_RETRY; } + readAgainPending = false; LoadResult result; try { - result = loader.load(); + result = loader.load(retryDelayMillis > 0 ? Duration.ofMillis(retryDelayMillis) : Duration.ZERO); + } catch (EmptiedFileException e) { + return readAgainLater(e); } catch (FileDataException e) { return fail(e); } // Close may have happened while the files were being read. Deliver nothing in that case. if (closed.get()) { - return true; + return NO_RETRY; } // A success right after a failure must apply even when the content is unchanged since the @@ -296,7 +380,8 @@ private boolean reload() { boolean recovering = lastErrorMessage != null; byte[] hash = result.getContentHash(); if (skipUnchanged && !recovering && hash != null && Arrays.equals(hash, lastGoodHash)) { - return true; + loader.accepted(result); + return NO_RETRY; } // Nothing is remembered until the consumer has accepted the result. A result that the // consumer rejects must not become the baseline that skip-unchanged compares against, and @@ -308,27 +393,44 @@ private boolean reload() { } lastErrorMessage = null; lastGoodHash = hash; - return true; + loader.accepted(result); + return NO_RETRY; + } + } + + // Called inside the reload lock. The loader held back files that it found empty: a file whose + // content is in effect may be in the middle of an in-place rewrite, a writer that truncates the file + // and then writes the new content. Whatever the files' format, the load keeps the current data and + // arms the retry without reporting a failure, for the time that the loader asks for. The change + // signal for the write starts a load sooner. The loader decides how long a file can be held back, and + // holds nothing back when retries are off, because nothing would read the file again. A closed + // reloader arms no retry. + private long readAgainLater(EmptiedFileException e) { + for (Path path : e.getPaths()) { + logger.debug("File {} is empty, possibly while it is being rewritten; reading it again before applying it", path); } + readAgainPending = true; + return e.getRetryAfterNanos(); } - private boolean fail(FileDataException e) { - // Close may have happened while the files were being read. Deliver nothing and report success - // so that no retry is armed. + // Reports a failure and returns when to retry. + private long fail(FileDataException e) { + // Close may have happened while the files were being read. Deliver nothing and arm no retry. if (closed.get()) { - return true; + return NO_RETRY; } + long retryAfterNanos = retryDelayMillis > 0 ? TimeUnit.MILLISECONDS.toNanos(retryDelayMillis) : NO_RETRY; // With automatic retries, a persistent failure would repeat the same log entry and the same // callback on every attempt. Repeats of an identical failure are logged at debug level and do // not call onError again. A consumer therefore sees one report per distinct failure. String description = describe(e); if (description.equals(lastErrorMessage)) { logger.debug("Unable to load flags: {}", description); - return false; + return retryAfterNanos; } lastErrorMessage = description; logger.error("Unable to load flags: {}", description); handler.onError(e); - return false; + return retryAfterNanos; } } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileDataSourceParsing.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileDataSourceParsing.java index 56fb4f22..179661f5 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileDataSourceParsing.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileDataSourceParsing.java @@ -2,7 +2,7 @@ import com.google.gson.Gson; import com.google.gson.JsonElement; -import com.google.gson.JsonObject; +import com.google.gson.JsonIOException; import com.google.gson.JsonSyntaxException; import com.launchdarkly.sdk.LDValue; import com.launchdarkly.sdk.ObjectBuilder; @@ -12,6 +12,7 @@ import org.yaml.snakeyaml.Yaml; import org.yaml.snakeyaml.constructor.SafeConstructor; import org.yaml.snakeyaml.error.YAMLException; +import org.yaml.snakeyaml.reader.UnicodeReader; import org.yaml.snakeyaml.representer.Representer; import java.io.ByteArrayInputStream; @@ -19,6 +20,8 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; +import java.nio.file.Path; +import java.util.Locale; import java.util.Map; import static com.launchdarkly.sdk.server.DataModel.FEATURES; @@ -64,6 +67,13 @@ public String getDescription() { * parse the flags or segments at this level; that will be done by {@link FlagFactory}. */ static final class FlagFileRep { + /** + * What a parser returns for content that holds no document, such as YAML with only a comment or a + * document marker. It has no entries. The override source compares a result with it by identity, + * so it must not be modified. + */ + static final FlagFileRep NO_DOCUMENT = new FlagFileRep(); + Map flags; Map flagValues; Map segments; @@ -91,7 +101,43 @@ public static FlagFileParser selectForContent(byte[] data) { Reader r = new InputStreamReader(new ByteArrayInputStream(data)); return detectJson(r) ? JsonParserHolder.getInstance() : YamlParserHolder.getInstance(); } - + + /** + * Chooses the parser by the extension of the file name: a file whose name ends in {@code .yaml} + * or {@code .yml}, in any case, is parsed as YAML, and any other file is parsed as JSON. The + * override source uses this. The file data source keeps choosing by content, with + * {@link #selectForContent(byte[])}. + * + * @param path the file + * @return the parser for the file + */ + static FlagFileParser selectForPath(Path path) { + Path fileName = path.getFileName(); + String name = fileName == null ? "" : fileName.toString().toLowerCase(Locale.ROOT); + return name.endsWith(".yaml") || name.endsWith(".yml") + ? YamlParserHolder.getInstance() : JsonParserHolder.getInstance(); + } + + /** + * Parses the content of a file with the parser that {@link #selectForPath(Path)} chooses. The + * text encoding is detected from a byte order mark, UTF-8, UTF-16 or UTF-32, and is UTF-8 + * otherwise, for both formats; the YAML parser does that itself. The override source uses this. + * The file data source keeps reading JSON in the platform's default encoding. + * + * @param path the file + * @param data its content + * @return the parsed content + * @throws FileDataException if the content cannot be decoded or parsed + * @throws IOException if the content cannot be read + */ + static FlagFileRep parseForPath(Path path, byte[] data) throws FileDataException, IOException { + FlagFileParser parser = selectForPath(path); + if (parser instanceof JsonFlagFileParser) { + return ((JsonFlagFileParser) parser).parse(new UnicodeReader(new ByteArrayInputStream(data))); + } + return parser.parse(new ByteArrayInputStream(data)); + } + private static boolean detectJson(Reader r) { // A valid JSON file for our purposes must be an object, i.e. it must start with '{' while (true) { @@ -119,11 +165,40 @@ static final class JsonFlagFileParser extends FlagFileParser { @Override public FlagFileRep parse(InputStream input) throws FileDataException, IOException { + return parse(new InputStreamReader(input)); + } + + /** + * Parses JSON from text that is already decoded. + * + * @param reader the text + * @return the parsed content + * @throws FileDataException if the text is not a JSON object, or cannot be decoded + * @throws IOException if the text cannot be read + */ + FlagFileRep parse(Reader reader) throws FileDataException, IOException { + JsonElement tree; try { - return parseJson(gson.fromJson(new InputStreamReader(input), JsonElement.class)); + tree = gson.fromJson(reader, JsonElement.class); } catch (JsonSyntaxException e) { throw new FileDataException("cannot parse JSON", e); - } + } catch (JsonIOException e) { + // For example, bytes that are not valid in the detected encoding. + throw new FileDataException("cannot read JSON", e); + } + if (tree == null) { + // Gson finds no value in a file that is empty or white space only, or, since it reads + // leniently, one that holds only comments. That is not a JSON document. A file that is being + // rewritten in place can be empty for a moment, and the failure keeps its last good data + // until it is complete. The file data source never gets here: it parses a file as JSON only + // when the file starts with '{'. + throw new FileDataException("the file has no content, which is not a JSON document", null); + } + if (!tree.isJsonObject()) { + // A document such as null or [] holds no flags. The file data source never gets here either. + throw new FileDataException("the JSON document is not an object", null); + } + return parseJson(tree); } public FlagFileRep parseJson(JsonElement tree) throws FileDataException, IOException { @@ -174,8 +249,12 @@ public FlagFileRep parse(InputStream input) throws FileDataException, IOExceptio } catch (YAMLException e) { throw new FileDataException("unable to parse YAML", e); } - JsonElement jsonRoot = root == null ? new JsonObject() : gson.toJsonTree(root); - return jsonFileParser.parseJson(jsonRoot); + if (root == null) { + // Empty content, or content with only comments or document markers, holds no document. It has + // no entries. + return FlagFileRep.NO_DOCUMENT; + } + return jsonFileParser.parseJson(gson.toJsonTree(root)); } } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceBuilder.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceBuilder.java new file mode 100644 index 00000000..4d056392 --- /dev/null +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceBuilder.java @@ -0,0 +1,186 @@ +package com.launchdarkly.sdk.server.integrations; + +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.sdk.server.integrations.FileOverrides.ChangeDetection; +import com.launchdarkly.sdk.server.subsystems.ClientContext; +import com.launchdarkly.sdk.server.subsystems.ComponentConfigurer; +import com.launchdarkly.sdk.server.subsystems.OverrideSource; + +import java.nio.file.FileSystems; +import java.nio.file.InvalidPathException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * Configures the file-based override source. Obtain an instance with {@link FileOverrides#source()}, + * call {@link #filePaths(String...)} or {@link #filePaths(Path...)} to specify the files, adjust + * any other options, and pass the builder to + * {@link DataSystemBuilder#overrides(ComponentConfigurer)}. + *

+ * For more details, see {@link FileOverrides}. + *

+ * Flag overrides are currently experimental and subject to change. + * + * @since 7.18.0 + */ +public final class FileOverrideSourceBuilder implements ComponentConfigurer { + private final List filePaths = new ArrayList<>(); + private FileData.DuplicateKeysHandling duplicateKeysHandling = FileData.DuplicateKeysHandling.FAIL; + private ChangeDetection changeDetection = ChangeDetection.POLLING; + private Duration pollInterval = FileOverrides.DEFAULT_POLL_INTERVAL; + + FileOverrideSourceBuilder() {} + + /** + * Adds any number of files to load overrides from, specifying each file path as a string. The + * order is significant: it determines which file wins under the duplicate keys handling when the + * same key appears in more than one file. + *

+ * A file whose name ends in {@code .yaml} or {@code .yml}, in any case, is parsed as YAML, and any + * other file is parsed as JSON. A file is read as UTF-8, or in the encoding that a byte order mark + * names. + *

+ * Flag overrides are currently experimental and subject to change. + * + * @param filePaths path(s) to the file(s); may be absolute or relative to the current working directory + * @return the same builder + * @throws InvalidPathException if one of the parameters is not a valid file path + */ + public FileOverrideSourceBuilder filePaths(String... filePaths) throws InvalidPathException { + for (String p : filePaths) { + this.filePaths.add(Paths.get(p)); + } + return this; + } + + /** + * Adds any number of files to load overrides from, specifying each file path as a Path. The + * order is significant: it determines which file wins under the duplicate keys handling when the + * same key appears in more than one file. + *

+ * A file whose name ends in {@code .yaml} or {@code .yml}, in any case, is parsed as YAML, and any + * other file is parsed as JSON. A file is read as UTF-8, or in the encoding that a byte order mark + * names. + *

+ * Flag overrides are currently experimental and subject to change. + * + * @param filePaths path(s) to the file(s); may be absolute or relative to the current working directory + * @return the same builder + */ + public FileOverrideSourceBuilder filePaths(Path... filePaths) { + for (Path p : filePaths) { + this.filePaths.add(p); + } + return this; + } + + /** + * Specifies how to handle the same key appearing in more than one file. + *

+ * With {@link FileData.DuplicateKeysHandling#FAIL}, the default, a reload fails if the same flag or + * segment key appears in more than one file, and the previously loaded overrides stay in effect. + * With {@link FileData.DuplicateKeysHandling#IGNORE}, the entry from the first configured file that + * defines the key is kept and the others are discarded. A null value selects the default. + *

+ * Flag overrides are currently experimental and subject to change. + * + * @param duplicateKeysHandling specifies how to handle duplicate keys + * @return the same builder + */ + public FileOverrideSourceBuilder duplicateKeysHandling(FileData.DuplicateKeysHandling duplicateKeysHandling) { + this.duplicateKeysHandling = duplicateKeysHandling == null + ? FileData.DuplicateKeysHandling.FAIL : duplicateKeysHandling; + return this; + } + + /** + * Selects how the source detects file changes. The default is {@link ChangeDetection#POLLING}. + * The two modes are alternatives, so setting one replaces the other. {@link ChangeDetection#WATCHING} + * requires every file path to be on the default file system; a path of another file system provider, + * such as a zip file system, can be polled but not watched, and {@link #build(ClientContext)} rejects it. + *

+ * Flag overrides are currently experimental and subject to change. + * + * @param changeDetection the change detection mode + * @return the same builder + */ + public FileOverrideSourceBuilder changeDetection(ChangeDetection changeDetection) { + this.changeDetection = changeDetection; + return this; + } + + /** + * Sets the interval between examinations of the files in {@link ChangeDetection#POLLING} mode. + * {@link ChangeDetection#WATCHING} mode ignores it. The default is + * {@link FileOverrides#DEFAULT_POLL_INTERVAL}. An interval below + * {@link FileOverrides#MINIMUM_POLL_INTERVAL} is raised to the minimum. + *

+ * Flag overrides are currently experimental and subject to change. + * + * @param pollInterval the polling interval + * @return the same builder + */ + public FileOverrideSourceBuilder pollInterval(Duration pollInterval) { + this.pollInterval = pollInterval; + return this; + } + + /** + * Called internally by the SDK to create the override source. + *

+ * Flag overrides are currently experimental and subject to change. + * + * @param context the client context + * @return the override source + * @throws IllegalArgumentException if no file paths were specified, or the change detection mode + * or the poll interval is null + */ + @Override + public OverrideSource build(ClientContext context) { + if (filePaths.isEmpty()) { + throw new IllegalArgumentException("no file paths were specified for the file-based override source"); + } + if (changeDetection == null) { + throw new IllegalArgumentException("a change detection mode is required for the file-based override source"); + } + if (pollInterval == null) { + throw new IllegalArgumentException("a poll interval is required for the file-based override source"); + } + // Change notifications come from the default file system's watch service, which cannot watch a + // path of another file system provider, such as a zip file system. That is a configuration + // error, reported here rather than when the source starts. Polling reads such a path through its + // own provider. + if (changeDetection == ChangeDetection.WATCHING) { + for (Path p : filePaths) { + if (!p.getFileSystem().equals(FileSystems.getDefault())) { + throw new IllegalArgumentException( + "the file-based override source can watch only files on the default file system; use polling for " + p); + } + } + } + LDLogger logger = context.getBaseLogger().subLogger("FileOverrideSource"); + + // A path listed more than once would make every entry of that file a duplicate of itself, so + // the repeats are dropped and the first occurrence keeps its place in the order. + Set absolutePaths = new LinkedHashSet<>(); + for (Path p : filePaths) { + absolutePaths.add(p.toAbsolutePath().normalize()); + } + if (absolutePaths.size() != filePaths.size()) { + logger.warn("File paths list the same file more than once; the repeats are ignored"); + } + + Duration effectivePollInterval = pollInterval; + if (changeDetection == ChangeDetection.POLLING && pollInterval.compareTo(FileOverrides.MINIMUM_POLL_INTERVAL) < 0) { + logger.warn("Poll interval {} is below the minimum; using {}", pollInterval, FileOverrides.MINIMUM_POLL_INTERVAL); + effectivePollInterval = FileOverrides.MINIMUM_POLL_INTERVAL; + } + + return new FileOverrideSourceImpl(new ArrayList<>(absolutePaths), duplicateKeysHandling, changeDetection, effectivePollInterval, logger); + } +} diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceImpl.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceImpl.java new file mode 100644 index 00000000..aeae0563 --- /dev/null +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceImpl.java @@ -0,0 +1,207 @@ +package com.launchdarkly.sdk.server.integrations; + +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.logging.LogValues; +import com.launchdarkly.sdk.server.integrations.FileDataSourceParsing.FileDataException; +import com.launchdarkly.sdk.server.integrations.FileOverrides.ChangeDetection; +import com.launchdarkly.sdk.server.integrations.OverrideFileLoader.FileSummary; +import com.launchdarkly.sdk.server.integrations.OverrideFileLoader.LoadResult; +import com.launchdarkly.sdk.server.subsystems.OverrideSink; +import com.launchdarkly.sdk.server.subsystems.OverrideSource; + +import java.io.IOException; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * The file-based override source. It reads one or more files in the file data source document + * format, merges them in the configured order, and supplies the result to the override sink as + * one snapshot. It reloads when the files change, by polling or by watching. + *

+ * A configured file that does not exist contributes no overrides. A file that exists but cannot + * be read or parsed fails that reload: the sink is not called, so the last good overrides stay in + * effect, and the reload is retried. A file whose content is in effect and is found empty may be in + * the middle of an in-place rewrite, so it is held back until it has stayed empty, without being + * written, for the retry delay, and for at most five retry delays from the first load that held it + * back. + */ +final class FileOverrideSourceImpl implements OverrideSource { + /** + * Creates the watcher for watching mode. Tests substitute one that fails. + */ + interface WatcherFactory { + FileDataWatcher create(List paths, LDLogger logger) throws IOException; + } + + private final List paths; + private final ChangeDetection changeDetection; + private final Duration pollInterval; + private final Duration retryDelay; + private final LDLogger logger; + private final OverrideFileLoader loader; + private final WatcherFactory watcherFactory; + + private volatile FileDataReloader reloader; + private volatile FileDataPoller poller; + private volatile FileDataWatcher watcher; + private final AtomicBoolean closed = new AtomicBoolean(false); + + FileOverrideSourceImpl( + List paths, + FileData.DuplicateKeysHandling duplicateKeysHandling, + ChangeDetection changeDetection, + Duration pollInterval, + LDLogger logger + ) { + this(paths, duplicateKeysHandling, changeDetection, pollInterval, FileDataReloader.DEFAULT_RETRY_DELAY, logger, + FileDataWatcher::create); + } + + /** + * Creates a source with a specific retry delay and watcher factory. Visible for tests. + */ + FileOverrideSourceImpl( + List paths, + FileData.DuplicateKeysHandling duplicateKeysHandling, + ChangeDetection changeDetection, + Duration pollInterval, + Duration retryDelay, + LDLogger logger, + WatcherFactory watcherFactory + ) { + this.paths = new ArrayList<>(paths); + this.changeDetection = changeDetection; + this.pollInterval = pollInterval; + this.retryDelay = retryDelay; + this.logger = logger; + this.loader = new OverrideFileLoader(paths, duplicateKeysHandling); + this.watcherFactory = watcherFactory; + } + + /** + * Performs the initial load synchronously, so overrides present in the files are in effect when + * the client constructor returns, then starts change detection. A file that does not exist yet + * contributes no overrides. A file that cannot be read or parsed is not fatal: the client runs + * with the last good overrides, the failure is logged, and the retry, plus the change signal, + * recovers once the file is readable. + */ + @Override + public void start(OverrideSink sink) { + reloader = new FileDataReloader( + loader, + new FileDataReloader.Handler() { + @Override + public void apply(LoadResult result) { + sink.setOverrides(result.getData()); + logOverridesInEffect(result); + } + + @Override + public void onError(FileDataException e) { + // The reloader has logged the failure. The last good overrides stay in effect. + } + }, + logger, + FileDataReloader.DEFAULT_DEBOUNCE_DELAY, + retryDelay, + true + ); + reloader.reloadNow(); + + if (closed.get()) { + return; + } + switch (changeDetection) { + case WATCHING: + try { + FileDataWatcher w = watcherFactory.create(paths, logger); + watcher = w; + w.start(reloader::trigger); + } catch (IOException e) { + // The watch service itself cannot be created, for example because the limit on inotify + // instances is reached. Without change detection a later edit would never take effect, so + // the source polls the files instead, no more often than polling mode allows. + Duration interval = pollInterval.compareTo(FileOverrides.MINIMUM_POLL_INTERVAL) < 0 + ? FileOverrides.MINIMUM_POLL_INTERVAL : pollInterval; + logger.error("Unable to watch override files; polling them instead every {}: {}", interval, + LogValues.exceptionSummary(e)); + logger.debug(LogValues.exceptionTrace(e)); + poller = new FileDataPoller(paths, interval, reloader::trigger, logger); + } + break; + case POLLING: + default: + poller = new FileDataPoller(paths, pollInterval, reloader::trigger, logger); + break; + } + } + + /** + * Returns the interval between file examinations in polling mode. Visible for tests. + * + * @return the effective poll interval + */ + Duration getPollInterval() { + return pollInterval; + } + + // Reports the overrides now in effect and the file each came from. The reloader applies a + // snapshot only when the content changed, so this logs each change once. + private void logOverridesInEffect(LoadResult result) { + List details = new ArrayList<>(result.getFiles().size()); + for (FileSummary file : result.getFiles()) { + if (!file.isPresent()) { + details.add(file.getPath() + ": absent"); + } else if (file.getFlags() == 0 && file.getSegments() == 0) { + details.add(file.getPath() + ": no entries"); + } else { + details.add(file.getPath() + ": " + countsText(file.getFlags(), file.getSegments())); + } + } + String fileDetails = String.join("; ", details); + if (result.getFlagCount() == 0 && result.getSegmentCount() == 0) { + logger.info("Flag overrides: none in effect ({})", fileDetails); + return; + } + logger.info("Flag overrides in effect: {} ({})", countsText(result.getFlagCount(), result.getSegmentCount()), + fileDetails); + } + + // Formats flag and segment counts, for example "2 flags, 1 segment". + static String countsText(int flags, int segments) { + List parts = new ArrayList<>(2); + if (flags > 0) { + parts.add(pluralize(flags, "flag")); + } + if (segments > 0) { + parts.add(pluralize(segments, "segment")); + } + return String.join(", ", parts); + } + + private static String pluralize(int count, String noun) { + return count == 1 ? "1 " + noun : count + " " + noun + "s"; + } + + @Override + public void close() { + if (closed.getAndSet(true)) { + return; + } + FileDataWatcher w = watcher; + if (w != null) { + w.close(); + } + FileDataPoller p = poller; + if (p != null) { + p.close(); + } + FileDataReloader r = reloader; + if (r != null) { + r.close(); + } + } +} diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrides.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrides.java new file mode 100644 index 00000000..02499f49 --- /dev/null +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrides.java @@ -0,0 +1,123 @@ +package com.launchdarkly.sdk.server.integrations; + +import java.time.Duration; + +/** + * Integration between the LaunchDarkly SDK and file-based flag overrides. + *

+ * Flag overrides are currently experimental and subject to change. + *

+ * Overrides are flag and segment definitions that take precedence over data received from + * LaunchDarkly at evaluation time, on a per-key basis. They exist for resilience during an + * incident. An operator can force one or more flags to a known state on a running application, + * whether or not the application can reach LaunchDarkly. The override stays in effect until the + * operator removes it. Flags not present in the override data are completely unaffected. + *

+ * This class provides one override source: {@link #source()}, which reads overrides from local + * files and reloads them as the files change. Configure it with the data system builder: + * 


+ *     LDConfig config = new LDConfig.Builder()
+ *         .dataSystem(Components.dataSystem().defaultMode()
+ *             .overrides(FileOverrides.source().filePaths("/etc/launchdarkly/overrides.json")))
+ *         .build();
+ *
+ *

+ * An evaluation that an override affects is marked. The marking is direct or transitive: it + * applies when the evaluated flag, a prerequisite at any depth, or a segment read during the + * evaluation came from the override layer. {@link com.launchdarkly.sdk.EvaluationReason#isOverrideAffected()} + * reports the marking. Marked evaluations appear in analytics summary events only, under separate + * counters, so LaunchDarkly can distinguish them. They produce no individual evaluation events. + * + * @see FileOverrideSourceBuilder + * @see DataSystemBuilder#overrides(com.launchdarkly.sdk.server.subsystems.ComponentConfigurer) + * @since 7.18.0 + */ +public abstract class FileOverrides { + /** + * Selects how the file-based override source learns that a file changed. The two modes are + * alternatives. Flag overrides are currently experimental and subject to change. + * + * @see FileOverrideSourceBuilder#changeDetection(ChangeDetection) + */ + public enum ChangeDetection { + /** + * The source examines the files on a fixed interval and reloads when the modification time or + * the size of a file changes. Polling works on every file system, including network mounts and + * directories whose contents are swapped through symbolic links, as Kubernetes does for mounted + * ConfigMaps. It is the default. + * Flag overrides are currently experimental and subject to change. + */ + POLLING, + + /** + * The source reloads in response to file system change notifications. It reacts faster than + * polling. It depends on notifications, which some file systems do not deliver reliably. + * Flag overrides are currently experimental and subject to change. + */ + WATCHING + } + + /** + * The interval at which the file source examines the files for changes in + * {@link ChangeDetection#POLLING} mode when no interval was specified. Because the source reads + * local files rather than contacting a service, a short interval keeps an override responsive + * during an incident at negligible cost. + * Flag overrides are currently experimental and subject to change. + */ + public static final Duration DEFAULT_POLL_INTERVAL = Duration.ofSeconds(1); + + /** + * The shortest allowed polling interval. A configured interval below this is raised to it. The + * minimum exists only to prevent a pathological tight loop over the file system. + * Flag overrides are currently experimental and subject to change. + */ + public static final Duration MINIMUM_POLL_INTERVAL = Duration.ofSeconds(1); + + private FileOverrides() {} + + /** + * Creates a {@link FileOverrideSourceBuilder} for a file-based override source. The source reads + * flag and segment overrides from one or more local files and reloads them as the files change. + *

+ * The files use the same document format as the file data source (see {@link FileData}): each + * file is a JSON or YAML document with optional {@code "flags"}, {@code "flagValues"}, and + * {@code "segments"} members. A file whose name ends in {@code .yaml} or {@code .yml}, in any case, + * is YAML, and any other file is JSON; the YAML parser does not accept every JSON document, so give + * a JSON file a name that does not end in {@code .yaml} or {@code .yml}. A file is read as UTF-8, + * or in the encoding that a byte order mark names. A {@code "flagValues"} entry is expanded into a + * full flag definition that is on and serves the given value as its single variation by fallthrough + * for every context. Such a flag is not available to client-side SDKs, so + * {@link com.launchdarkly.sdk.server.LDClient#allFlagsState} with + * {@link com.launchdarkly.sdk.server.FlagsStateOption#CLIENT_SIDE_ONLY} leaves it out, even when + * the flag it overrides is available to them. To override such a flag, define it in + * {@code "flags"} with {@code "clientSide": true}. + * When multiple files are configured, their entries are combined in the configured order, and + * the duplicate keys handling decides which file wins for a key that appears more than once. + *

+ * A reload replaces the entire override set, so removing an entry from the files removes the + * override. A configured file that does not exist contributes no overrides, so deleting every file + * removes them all. A file that exists but cannot be read or parsed makes the reload fail: the + * overrides in effect stay, the source logs the failure, and it recovers on its own once the file + * can be read again. + *

+ * Files can be rewritten in place: if the source finds a file empty while it is being rewritten, it + * keeps the overrides in effect until the new content is written. To remove a file's overrides, + * delete the file or write a document with no entries, such as {}. An empty JSON file + * is not a valid document. For a YAML file, replacing its contents in one operation, such as by + * renaming a fully written temporary file over it, avoids briefly applying a partly written file. + *

+ * Whenever the set of overrides in effect changes, including at startup, the source logs the + * overrides in effect and what each configured file supplied, at Info level. + *

+ * By default the source polls the files for changes once per second. See + * {@link FileOverrideSourceBuilder#changeDetection(ChangeDetection)} and + * {@link FileOverrideSourceBuilder#pollInterval(Duration)}. + *

+ * Flag overrides are currently experimental and subject to change. + * + * @return a builder for the file-based override source + */ + public static FileOverrideSourceBuilder source() { + return new FileOverrideSourceBuilder(); + } +} diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/OverrideFileLoader.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/OverrideFileLoader.java index ece5b8b7..f58488c5 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/OverrideFileLoader.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/OverrideFileLoader.java @@ -2,6 +2,7 @@ import com.google.common.collect.ImmutableList; import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.server.integrations.FileDataReloader.EmptiedFileException; import com.launchdarkly.sdk.server.integrations.FileDataSourceParsing.FileDataException; import com.launchdarkly.sdk.server.integrations.FileDataSourceParsing.FlagFactory; import com.launchdarkly.sdk.server.integrations.FileDataSourceParsing.FlagFileParser; @@ -10,19 +11,25 @@ import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor; import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.KeyedItems; +import org.yaml.snakeyaml.reader.UnicodeReader; + import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.Reader; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.time.Duration; import java.util.AbstractMap; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.LongSupplier; import static com.launchdarkly.sdk.server.DataModel.FEATURES; import static com.launchdarkly.sdk.server.DataModel.SEGMENTS; @@ -33,22 +40,55 @@ * This loader is separate from the file data source's loader because the override source has * different rules: a configured file that does not exist contributes no entries, entries keep the * versions that the documents specify, a value-only entry becomes a flag that is on and serves the - * value by fallthrough, and every failure to read or parse a file is reported as a file data error. - * The file data source keeps its own behavior. + * value by fallthrough, the format of a file is chosen by its extension, and every failure to read or + * parse a file is reported as a file data error. The file data source keeps its own behavior. + *

+ * Whether a file with no content is valid depends on its format, like the rest of its content. A JSON + * file with no content is not a JSON document, so it fails to parse. A YAML file with no content is a + * document with no entries. + *

+ * A file that is rewritten in place is empty for a moment, and a load can find it so. A file counts + * as empty when it holds only white space, or when its parser reads its content as no document, as the + * YAML parser does for a part of a file that holds only a comment or a document marker. The loader + * remembers, for each file, whether it contributed content to the data in effect, and since when it + * has been found empty without being written. A file whose content is in effect and that is found + * empty is held back, not parsed: {@link #load(Duration)} still reads the other files, so that files + * emptied together are held back from the same load, and then throws {@link EmptiedFileException}. + * The data in effect stays, and the reloader reads the files again when the exception says. A file is + * held back until it has stayed empty, without being written, for one retry delay, and for at most + * five retry delays from the first load that held it back, so that a file that keeps being rewritten + * cannot hold the changes back for longer. The bound is per file, and it starts over at a load that + * holds nothing back. A file that never contributed content, such as a placeholder, and a file whose empty state + * is in effect are parsed right away. The reloader calls {@link #load(Duration)} and + * {@link #accepted(LoadResult)} one call at a time, so this state needs no synchronization. */ -final class OverrideFileLoader { +final class OverrideFileLoader implements FileDataReloader.Loader { + /** + * How many retry delays a file can be held back for, from the first load that held it back. + */ + static final int MAX_HOLD_RETRY_DELAYS = 5; + + /** + * Reads a file. Tests substitute one that takes time. + */ + interface FileReader { + byte[] read(Path path) throws IOException; + } + /** * Describes one configured file after a load. */ static final class FileSummary { private final Path path; private final boolean present; + private final boolean hasContent; private final int flags; private final int segments; - FileSummary(Path path, boolean present, int flags, int segments) { + FileSummary(Path path, boolean present, boolean hasContent, int flags, int segments) { this.path = path; this.present = present; + this.hasContent = hasContent; this.flags = flags; this.segments = segments; } @@ -64,6 +104,14 @@ boolean isPresent() { return present; } + /** + * False when the file does not exist, is empty, holds only white space or a byte order mark, or + * holds content that its parser reads as no document, such as YAML with only a comment. + */ + boolean hasContent() { + return hasContent; + } + /** * The number of flag entries the merge kept from this file. An entry dropped by the duplicate keys * handling is not counted. @@ -132,8 +180,32 @@ byte[] getContentHash() { } } + /** + * A file whose content is in effect and that loads have found empty. + * Times are from the monotonic clock. + */ + private static final class EmptyFile { + // Since when the file has been empty without being written, or null after a load read it with + // content, even content that fails to parse, or found it missing. + Long sinceNanos; + // The state of the file then: its modification time and size, read through any symbolic link. It + // is only compared for equality. + FileDataPoller.FileState state; + // When a load first held the file back, or null if no load has since the last load that got to the + // file and held nothing back. Loads that hold another file back keep it, so that a file that keeps + // being rewritten stays bounded. + Long firstHeldNanos; + } + private final List paths; private final FileData.DuplicateKeysHandling duplicateKeysHandling; + private final LongSupplier nanoClock; + // Whether each configured file contributed content to the data in effect. Updated only by + // accepted(LoadResult). A file that is not listed did not. + private final Map appliedHadContent = new HashMap<>(); + private final FileReader reader; + // The files whose content is in effect and that loads have found empty. + private final Map emptyFiles = new HashMap<>(); /** * Creates a loader. @@ -142,84 +214,334 @@ byte[] getContentHash() { * @param duplicateKeysHandling what to do when the same key appears in more than one file */ OverrideFileLoader(List paths, FileData.DuplicateKeysHandling duplicateKeysHandling) { + this(paths, duplicateKeysHandling, System::nanoTime); + } + + /** + * Creates a loader with a specific monotonic clock. Visible for tests. + */ + OverrideFileLoader(List paths, FileData.DuplicateKeysHandling duplicateKeysHandling, + LongSupplier nanoClock) { + this(paths, duplicateKeysHandling, nanoClock, Files::readAllBytes); + } + + /** + * Creates a loader with a specific monotonic clock and file reader. Visible for tests. + */ + OverrideFileLoader(List paths, FileData.DuplicateKeysHandling duplicateKeysHandling, + LongSupplier nanoClock, FileReader reader) { this.paths = new ArrayList<>(paths); this.duplicateKeysHandling = duplicateKeysHandling; + this.nanoClock = nanoClock; + this.reader = reader; } /** * Reads every configured file in order and merges the entries. * + * @param retryDelay the reloader's retry delay, or zero if it does not retry. A file whose content is + * in effect and that is found empty is held back until it has stayed empty, without being written, + * for this long, and for at most {@link #MAX_HOLD_RETRY_DELAYS} times this from the first load + * that held it back. With zero, every file is parsed as it is. * @return the merged result with a summary of each file - * @throws FileDataException if a file that exists cannot be read or parsed, or the files cannot be merged + * @throws FileDataException if a file that exists cannot be read or parsed, or the files cannot be + * merged, also when another file is held back + * @throws EmptiedFileException if a file is held back and every other file loads */ - LoadResult load() throws FileDataException { + @Override + public LoadResult load(Duration retryDelay) throws FileDataException, EmptiedFileException { MessageDigest digest = newDigest(); - Map> data = new HashMap<>(); - List files = new ArrayList<>(paths.size()); - for (Path path : paths) { - byte[] raw; - try { - raw = Files.readAllBytes(path); - } catch (NoSuchFileException e) { - // Absence is a valid state, not a failure. - files.add(new FileSummary(path, false, 0, 0)); - continue; - } catch (IOException e) { - throw new FileDataException("file " + path + ": unable to read file", e); + List heldBack = new ArrayList<>(); + // The files that this load got to and did not hold back: read with content, parsed empty, found + // missing, or failing to read or parse. A load reads every configured file, also after a failure, + // so every file that is not held back is here. + List reached = new ArrayList<>(); + // The parsed documents of the files that are not held back, in configuration order. + List documents = new ArrayList<>(paths.size()); + // The first failure to read or parse a file. The other files are still read, so that each file + // is classified, and the load fails with this one after them. + FileDataException failure = null; + try { + for (Path path : paths) { + byte[] raw; + try { + raw = reader.read(path); + } catch (NoSuchFileException e) { + // Absence is a valid state, not a failure. + noteNotEmpty(path, reached); + updateDigest(digest, path, null); + documents.add(new ReadFile(path, null, false)); + continue; + } catch (IOException e) { + reached.add(path); + failure = first(failure, new FileDataException("file " + path + ": unable to read file", e)); + continue; + } + // Content that the parser reads as no document, such as YAML with only a comment, has no + // entries, and a read can see that much of a file that is being written. Content with only + // white space is invalid JSON, so it is parsed only when the file is not held back. + FlagFileRep document = null; + boolean hasContent = hasContent(raw); + if (hasContent) { + try { + document = parse(path, raw); + } catch (FileDataException e) { + noteNotEmpty(path, reached); + failure = first(failure, e); + continue; + } + hasContent = document != FlagFileRep.NO_DOCUMENT; + } + if (hasContent) { + noteNotEmpty(path, reached); + } else if (shouldHoldBack(path, retryDelay, nanoClock.getAsLong())) { + // The clock is read after this file's own read, so a slow read of another file does not + // shorten its delay. The other files are still read, so that each emptied file starts its + // delay in this load, and a failure of another file still fails the load. + heldBack.add(path); + continue; + } else { + reached.add(path); + } + // One read feeds both the digest and the parse, so the skip-unchanged digest can never disagree + // with the content that was applied. + updateDigest(digest, path, raw); + if (document == null) { + try { + document = parse(path, raw); + } catch (FileDataException e) { + failure = first(failure, e); + continue; + } + } + documents.add(new ReadFile(path, document, hasContent)); + } + + // A failure fails the load before a hold, and a hold comes before the merge. + if (failure != null) { + throw failure; + } + if (!heldBack.isEmpty()) { + throw new EmptiedFileException(heldBack, holdBack(heldBack, retryDelay, nanoClock.getAsLong())); + } + + // The entries are merged only after every file is read, so that a duplicate key or an entry + // that the data model rejects cannot keep a later file from being read. + Map> data = new HashMap<>(); + List files = new ArrayList<>(documents.size()); + for (ReadFile file : documents) { + if (file.document == null) { + files.add(new FileSummary(file.path, false, false, 0, 0)); + continue; + } + int[] added = addEntries(data, file.path, file.document); + files.add(new FileSummary(file.path, true, file.hasContent, added[0], added[1])); } - // One read feeds both the digest and the parse, so the skip-unchanged digest can never disagree - // with the content that was applied. - if (digest != null) { - digest.update(raw); - digest.update((byte) 0); + ImmutableList.Builder>> all = ImmutableList.builder(); + for (Map.Entry> e : data.entrySet()) { + all.add(new AbstractMap.SimpleEntry<>(e.getKey(), new KeyedItems<>(ImmutableList.copyOf(e.getValue().entrySet())))); } - int flagsAdded = 0; - int segmentsAdded = 0; - try { - FlagFileParser parser = FlagFileParser.selectForContent(raw); - FlagFileRep fileContents = parser.parse(new ByteArrayInputStream(raw)); - if (fileContents.flags != null) { - for (Map.Entry e : fileContents.flags.entrySet()) { - if (add(data, FEATURES, e.getKey(), FlagFactory.flagFromJson(e.getValue()))) { - flagsAdded++; - } + Map flags = data.get(FEATURES); + Map segments = data.get(SEGMENTS); + return new LoadResult(all.build(), files, flags == null ? 0 : flags.size(), + segments == null ? 0 : segments.size(), digest == null ? null : digest.digest()); + } finally { + if (heldBack.isEmpty()) { + // A load that holds nothing back, whether it is applied, skipped as unchanged, or fails, + // starts the bound over for every file that it got to, including one that failed: such a + // file is no longer held, so its next rewrite is a new one. + for (Path path : reached) { + EmptyFile empty = emptyFiles.get(path); + if (empty != null) { + empty.firstHeldNanos = null; } } - if (fileContents.flagValues != null) { - for (Map.Entry e : fileContents.flagValues.entrySet()) { - if (add(data, FEATURES, e.getKey(), FlagFactory.flagWithValue(e.getKey(), e.getValue(), 0))) { - flagsAdded++; - } + } + } + } + + /** + * A file that a load read and did not hold back. + */ + private static final class ReadFile { + final Path path; + // The parsed content, or null if the file does not exist. + final FlagFileRep document; + final boolean hasContent; + + ReadFile(Path path, FlagFileRep document, boolean hasContent) { + this.path = path; + this.document = document; + this.hasContent = hasContent; + } + } + + private static FileDataException first(FileDataException earlier, FileDataException e) { + return earlier != null ? earlier : e; + } + + // Adds a file to the skip-unchanged digest: its path, whether it exists, and its content. The format + // depends on the path, so the same bytes at another path can be different data. + private static void updateDigest(MessageDigest digest, Path path, byte[] raw) { + if (digest == null) { + return; + } + digest.update(path.toString().getBytes(StandardCharsets.UTF_8)); + digest.update((byte) 0); + digest.update((byte) (raw == null ? 0 : 1)); + if (raw != null) { + digest.update(raw); + digest.update((byte) 0); + } + } + + // For a file that a load read with content, including content that fails to parse, or found + // missing: if it was empty before, its empty state has ended, so its next emptying starts a new delay + // even where a coarse modification time makes it look the same. + private void noteNotEmpty(Path path, List reached) { + reached.add(path); + EmptyFile empty = emptyFiles.get(path); + if (empty != null) { + empty.sinceNanos = null; + } + } + + private static FlagFileRep parse(Path path, byte[] raw) throws FileDataException { + try { + return FlagFileParser.parseForPath(path, raw); + } catch (FileDataException e) { + throw new FileDataException("file " + path + ": " + e.getMessage(), e.getCause()); + } catch (IOException e) { + throw new FileDataException("file " + path + ": cannot read document", e); + } catch (RuntimeException e) { + throw new FileDataException("file " + path + ": cannot parse flag or segment data", e); + } + } + + // Adds the entries of one file and returns how many flags and segments the merge kept from it. + private int[] addEntries(Map> data, Path path, FlagFileRep document) + throws FileDataException { + int flagsAdded = 0; + int segmentsAdded = 0; + try { + if (document.flags != null) { + for (Map.Entry e : document.flags.entrySet()) { + if (add(data, FEATURES, e.getKey(), FlagFactory.flagFromJson(e.getValue()))) { + flagsAdded++; + } + } + } + if (document.flagValues != null) { + for (Map.Entry e : document.flagValues.entrySet()) { + if (add(data, FEATURES, e.getKey(), FlagFactory.flagWithValue(e.getKey(), e.getValue(), 0))) { + flagsAdded++; } } - if (fileContents.segments != null) { - for (Map.Entry e : fileContents.segments.entrySet()) { - if (add(data, SEGMENTS, e.getKey(), FlagFactory.segmentFromJson(e.getValue()))) { - segmentsAdded++; - } + } + if (document.segments != null) { + for (Map.Entry e : document.segments.entrySet()) { + if (add(data, SEGMENTS, e.getKey(), FlagFactory.segmentFromJson(e.getValue()))) { + segmentsAdded++; } } - } catch (FileDataException e) { - throw new FileDataException("file " + path + ": " + e.getMessage(), e.getCause()); - } catch (IOException e) { - throw new FileDataException("file " + path + ": cannot read document", e); - } catch (RuntimeException e) { - // A document can be valid JSON or YAML and still hold a flag or segment that the data model - // cannot accept. For the override source that is a parse failure of the file, so the last good - // overrides stay in effect. - throw new FileDataException("file " + path + ": cannot parse flag or segment data", e); } - files.add(new FileSummary(path, true, flagsAdded, segmentsAdded)); + } catch (FileDataException e) { + throw new FileDataException("file " + path + ": " + e.getMessage(), e.getCause()); + } catch (RuntimeException e) { + // A document can be valid JSON or YAML and still hold a flag or segment that the data model + // cannot accept. For the override source that is a parse failure of the file, so the last good + // overrides stay in effect. + throw new FileDataException("file " + path + ": cannot parse flag or segment data", e); + } + return new int[] { flagsAdded, segmentsAdded }; + } + + /** + * Records that the result of a load is now the data in effect: it was applied, or it was skipped + * because it is unchanged. Whether each file contributed content to the data in effect decides + * whether finding it empty later defers it. + * + * @param result the result of a load by this loader + */ + @Override + public void accepted(LoadResult result) { + for (FileSummary file : result.getFiles()) { + appliedHadContent.put(file.getPath(), file.hasContent()); + } + } + + // Decides whether a file that a load found empty is held back. A file whose content is in effect + // and that is empty now may be in the middle of an in-place rewrite: a writer that truncates the file + // and then writes the new content. Such a file is held back until it has stayed empty, without being + // written, for the retry delay. A change of its state means that it was written since a load last + // found it empty, for example rewritten and truncated again between two loads, so the delay starts + // over. A file that keeps being written while loads find it empty would be held back without end, so + // each file is held back for at most MAX_HOLD_RETRY_DELAYS retry delays from the first load that held + // it back. The bound is the file's own, so one file reaching it does not cut short another file's + // rewrite. It starts over only at a load that holds nothing back, for every file that load got to, + // including one that fails to read or parse, whatever the load's outcome. Loads that hold another file back keep it, which bounds files + // rewritten in turn or emptied in staggered turns, and a file that was read during a long failure + // of another file has a fresh bound when that failure ends. A file whose content is not in effect, + // such as a placeholder that has always been empty, or a file whose empty state was applied, is + // parsed right away. A zero retry delay, which means that the reloader does not retry, holds nothing + // back: nothing would read the file again. Elapsed times are differences of monotonic clock + // readings, which do not overflow. + private boolean shouldHoldBack(Path path, Duration retryDelay, long now) { + if (!Boolean.TRUE.equals(appliedHadContent.get(path))) { + return false; } + FileDataPoller.FileState state = FileDataPoller.observe(path); + EmptyFile empty = emptyFiles.computeIfAbsent(path, p -> new EmptyFile()); + if (empty.sinceNanos == null || !state.equals(empty.state)) { + empty.sinceNanos = now; + empty.state = state; + } + long delay = retryDelay.toNanos(); + if (now - empty.sinceNanos >= delay) { + // The file has stayed empty for the whole delay, so it is parsed as it is. Its bound starts over + // only at a load that holds nothing back, so that files that stay empty in staggered turns, each + // keeping the loads holding while the other settles, cannot hold the changes back without end. + return false; + } + if (empty.firstHeldNanos != null && now - empty.firstHeldNanos >= MAX_HOLD_RETRY_DELAYS * delay) { + return false; + } + if (empty.firstHeldNanos == null) { + empty.firstHeldNanos = now; + } + return true; + } - ImmutableList.Builder>> all = ImmutableList.builder(); - for (Map.Entry> e : data.entrySet()) { - all.add(new AbstractMap.SimpleEntry<>(e.getKey(), new KeyedItems<>(ImmutableList.copyOf(e.getValue().entrySet())))); + // Returns how long the reloader waits before it loads again: until the first of the held files has + // stayed empty for the retry delay or reaches its bound, plus a tenth of the retry delay, so that a + // timer that fires a little early still finds the time over. A change signal for a write starts a + // load sooner. + private long holdBack(List heldBack, Duration retryDelay, long now) { + long delay = retryDelay.toNanos(); + long remaining = Long.MAX_VALUE; + for (Path path : heldBack) { + EmptyFile empty = emptyFiles.get(path); + long delayLeft = delay - (now - empty.sinceNanos); + long boundLeft = MAX_HOLD_RETRY_DELAYS * delay - (now - empty.firstHeldNanos); + remaining = Math.min(remaining, Math.min(delayLeft, boundLeft)); + } + return Math.max(remaining, 0) + delay / 10; + } + + // A file with only white space, after a byte order mark if it has one, has no content. The text is + // decoded as the parsers decode it. Content that cannot be decoded is left to the parser to report. + private static boolean hasContent(byte[] raw) { + try (Reader text = new UnicodeReader(new ByteArrayInputStream(raw))) { + int c; + while ((c = text.read()) >= 0) { + if (!Character.isWhitespace(c)) { + return true; + } + } + return false; + } catch (IOException e) { + return true; } - Map flags = data.get(FEATURES); - Map segments = data.get(SEGMENTS); - return new LoadResult(all.build(), files, flags == null ? 0 : flags.size(), - segments == null ? 0 : segments.size(), digest == null ? null : digest.digest()); } // Adds an entry unless the key was already added. Reports whether it added the entry. diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/subsystems/DataSystemConfiguration.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/subsystems/DataSystemConfiguration.java index 34788d3c..50ecdd2e 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/subsystems/DataSystemConfiguration.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/subsystems/DataSystemConfiguration.java @@ -38,13 +38,14 @@ public enum DataStoreMode { private final ComponentConfigurer fDv1FallbackSynchronizer; private final ComponentConfigurer persistentStore; private final DataStoreMode persistentDataStoreMode; + private final ComponentConfigurer overrideSource; /** - * Creates an instance. + * Creates an instance without an override source. *

* This constructor is internal and should not be called by application code. *

- * + * * @param initializers see {@link #getInitializers()} * @param synchronizers see {@link #getSynchronizers()} * @param fDv1FallbackSynchronizer see {@link #getFDv1FallbackSynchronizer()} @@ -57,11 +58,37 @@ public DataSystemConfiguration( ComponentConfigurer fDv1FallbackSynchronizer, ComponentConfigurer persistentStore, DataStoreMode persistentDataStoreMode) { + this(initializers, synchronizers, fDv1FallbackSynchronizer, persistentStore, persistentDataStoreMode, null); + } + + /** + * Creates an instance with an optional override source. Flag overrides are currently experimental + * and subject to change. + *

+ * This constructor is internal and should not be called by application code. + *

+ * + * @param initializers see {@link #getInitializers()} + * @param synchronizers see {@link #getSynchronizers()} + * @param fDv1FallbackSynchronizer see {@link #getFDv1FallbackSynchronizer()} + * @param persistentStore see {@link #getPersistentStore()} + * @param persistentDataStoreMode see {@link #getPersistentDataStoreMode()} + * @param overrideSource see {@link #getOverrideSource()} + * @since 7.18.0 + */ + public DataSystemConfiguration( + ImmutableList> initializers, + ImmutableList> synchronizers, + ComponentConfigurer fDv1FallbackSynchronizer, + ComponentConfigurer persistentStore, + DataStoreMode persistentDataStoreMode, + ComponentConfigurer overrideSource) { this.initializers = initializers; this.synchronizers = synchronizers; this.fDv1FallbackSynchronizer = fDv1FallbackSynchronizer; this.persistentStore = persistentStore; this.persistentDataStoreMode = persistentDataStoreMode; + this.overrideSource = overrideSource; } /** @@ -107,11 +134,24 @@ public ComponentConfigurer getPersistentStore() { /** * The mode of operation for the persistent data store. - * + * * @return the persistent data store mode */ public DataStoreMode getPersistentDataStoreMode() { return persistentDataStoreMode; } + + /** + * An optional factory for creating an override source. The source supplies flag and segment + * overrides that take precedence over LaunchDarkly data at evaluation time. It is not part of + * the initializer or synchronizer pipeline and does not affect the client's initialization + * status. Flag overrides are currently experimental and subject to change. + * + * @return the override source configurer, or null + * @since 7.18.0 + */ + public ComponentConfigurer getOverrideSource() { + return overrideSource; + } } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/subsystems/EventProcessor.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/subsystems/EventProcessor.java index 2381b80f..3fb3a3c2 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/subsystems/EventProcessor.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/subsystems/EventProcessor.java @@ -57,6 +57,54 @@ void recordEvaluationEvent( boolean excludeFromSummaries, Long samplingRatio ); + + /** + * Records the action of evaluating a feature flag, with the override-affected marking. + *

+ * An override-affected evaluation is one that read at least one definition from the SDK's + * override store: the flag itself, a prerequisite at any depth, or a segment consulted during + * matching. Such an evaluation produces no individual feature event and no debug event, whatever + * the flag's configuration requests, and it is counted in a separate summary counter that + * carries the marker. The SDK keys this behavior on the marking alone, not on the reason. + *

+ * The default implementation ignores the marking and calls the method without it, so an + * existing custom implementation keeps working. Flag overrides are currently experimental and + * subject to change. + * + * @param context the evaluation context + * @param flagKey key of the feature flag that was evaluated + * @param flagVersion the version of the flag, or {@link #NO_VERSION} if the flag was not found + * @param variation the result variation index, or {@link EvaluationDetail#NO_VARIATION} if evaluation failed + * @param value the result value + * @param reason the evaluation reason, or null if the reason was not requested + * @param defaultValue the default value parameter for the evaluation + * @param prerequisiteOfFlagKey the key of the flag that this flag was evaluated as a prerequisite of, + * or null if this flag was evaluated for itself + * @param requireFullEvent true if full-fidelity analytics events should be sent for this flag + * @param debugEventsUntilDate if non-null, debug events are to be generated until this millisecond time + * @param excludeFromSummaries true if the event evaluation should not be included in summaries + * @param samplingRatio ratio used to control event sampling + * @param overrideAffected true if an override affected the evaluation + * @since 7.18.0 + */ + default void recordEvaluationEvent( + LDContext context, + String flagKey, + int flagVersion, + int variation, + LDValue value, + EvaluationReason reason, + LDValue defaultValue, + String prerequisiteOfFlagKey, + boolean requireFullEvent, + Long debugEventsUntilDate, + boolean excludeFromSummaries, + Long samplingRatio, + boolean overrideAffected + ) { + recordEvaluationEvent(context, flagKey, flagVersion, variation, value, reason, defaultValue, + prerequisiteOfFlagKey, requireFullEvent, debugEventsUntilDate, excludeFromSummaries, samplingRatio); + } /** * Registers an evaluation context, as when the SDK's {@code identify} method is called. diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/subsystems/OverrideSink.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/subsystems/OverrideSink.java new file mode 100644 index 00000000..b8fab1e5 --- /dev/null +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/subsystems/OverrideSink.java @@ -0,0 +1,36 @@ +package com.launchdarkly.sdk.server.subsystems; + +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.DataKind; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.KeyedItems; + +import java.util.Map; + +/** + * Receives the contents of the SDK's flag and segment override layer. The SDK implements this + * interface and passes it to {@link OverrideSource#start(OverrideSink)}. Override sources call + * it. They do not implement it. + *

+ * Flag overrides are currently experimental and subject to change. + * + * @since 7.18.0 + */ +public interface OverrideSink { + /** + * Replaces the entire override layer with the given flag and segment entries. An empty + * collection clears the layer. Each call is a full snapshot: entries absent from the call are + * removed from the layer. + *

+ * The data uses the SDK's standard data kinds, {@link com.launchdarkly.sdk.server.DataModel#FEATURES} + * and {@link com.launchdarkly.sdk.server.DataModel#SEGMENTS}, with item descriptors whose items + * are the SDK's flag and segment model objects, as produced by + * {@link DataKind#deserialize(String)}. Sources supply ordinary, fully parsed entities. The SDK + * itself marks the entries as overrides. The source's objects are never modified. + *

+ * This method is safe to call from any thread. The SDK serializes calls, and the new layer + * contents are visible to evaluations when the call returns. + * + * @param data the complete set of override entries, grouped by data kind + */ + void setOverrides(Iterable>> data); +} diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/subsystems/OverrideSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/subsystems/OverrideSource.java new file mode 100644 index 00000000..ba55d2b7 --- /dev/null +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/subsystems/OverrideSource.java @@ -0,0 +1,45 @@ +package com.launchdarkly.sdk.server.subsystems; + +import java.io.Closeable; + +/** + * Supplies flag and segment overrides that take precedence over LaunchDarkly data at evaluation + * time, on a per-key basis. Overrides exist for resilience during an incident. They let an operator + * force one or more flags to a known state on a running client, whether or not the client can + * reach LaunchDarkly. + *

+ * An override source is not a data source. It does not take part in the data system's initializer + * and synchronizer pipeline. The override layer that it populates has no effect on the client's + * initialization status, data availability, or data source status. + *

+ * To configure an override source, use + * {@link com.launchdarkly.sdk.server.integrations.DataSystemBuilder#overrides(ComponentConfigurer)}. + * The SDK closes the source when the client is closed. An exception thrown from {@link #close()} is + * logged and does not stop the client from shutting down. A snapshot that the source supplies after + * the client begins to close it is ignored. The SDK provides a file-based source; see + * {@link com.launchdarkly.sdk.server.integrations.FileOverrides}. + *

+ * Flag overrides are currently experimental and subject to change. + * + * @since 7.18.0 + */ +public interface OverrideSource extends Closeable { + /** + * Begins supplying overrides to the sink and returns without blocking on long-running work. + *

+ * Implementations typically perform an initial load synchronously, then push a full replacement + * snapshot to the sink whenever their backing data changes, until {@link #close()} is called. A + * failed load should leave the previously supplied layer untouched by not calling the sink. + *

+ * The SDK calls this method at most once, before any call to {@link #close()}, while the client + * is being constructed. This method is not intended to fail. A source that cannot load its data + * logs the failure and keeps retrying, as the SDK's file-based source does. A configuration error + * belongs in the {@link ComponentConfigurer#build(ClientContext)} that builds the source, where it + * is reported when the client is created. If this method does throw, the client still starts: the + * SDK logs the exception, closes the source, removes any snapshot the source supplied, and the + * client runs without overrides. + * + * @param sink receives override snapshots + */ + void start(OverrideSink sink); +} diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataModelTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataModelTest.java index 9cbbe9cd..eab1684c 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataModelTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataModelTest.java @@ -11,9 +11,20 @@ import com.launchdarkly.sdk.server.DataModel.SegmentRule; import com.launchdarkly.sdk.server.DataModel.Target; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor; + import org.junit.Test; +import static com.launchdarkly.sdk.server.ModelBuilders.flagBuilder; +import static com.launchdarkly.sdk.server.ModelBuilders.prerequisite; +import static com.launchdarkly.sdk.server.ModelBuilders.segmentBuilder; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; @SuppressWarnings("javadoc") public class DataModelTest { @@ -108,4 +119,82 @@ private Segment segmentWithAllZeroValuedFields() { // and no preprocessing has happened. return new Segment(); } + + @Test + public void flagAndSegmentAreNotOverridesByDefault() { + assertFalse(flagBuilder("f").build().isOverride()); + assertFalse(segmentBuilder("s").build().isOverride()); + assertFalse(((FeatureFlag) DataModel.FEATURES.deserialize("{\"key\":\"f\",\"version\":1}").getItem()).isOverride()); + assertFalse(((Segment) DataModel.SEGMENTS.deserialize("{\"key\":\"s\",\"version\":1}").getItem()).isOverride()); + } + + @Test + public void markedFlagCopyCarriesMarkerAndSharesDataWithoutMutatingSource() { + FeatureFlag source = flagBuilder("f").version(7).on(true).variations(LDValue.of("a"), LDValue.of("b")) + .fallthroughVariation(1).offVariation(0).prerequisites(prerequisite("p", 1)).trackEvents(true) + .debugEventsUntilDate(1000L).build(); + + FeatureFlag marked = source.markedAsOverride(); + + assertTrue(marked.isOverride()); + assertFalse(source.isOverride()); + assertNotSame(source, marked); + assertEquals(source.getKey(), marked.getKey()); + assertEquals(source.getVersion(), marked.getVersion()); + assertEquals(source.isOn(), marked.isOn()); + assertSame(source.getVariations(), marked.getVariations()); + assertSame(source.getPrerequisites(), marked.getPrerequisites()); + assertSame(source.getFallthrough(), marked.getFallthrough()); + assertEquals(source.getOffVariation(), marked.getOffVariation()); + assertEquals(source.isTrackEvents(), marked.isTrackEvents()); + assertEquals(source.getDebugEventsUntilDate(), marked.getDebugEventsUntilDate()); + assertSame(source.preprocessed, marked.preprocessed); + } + + @Test + public void markedFlagCopyOfUnpreprocessedFlagHasNoPreprocessing() { + FeatureFlag source = flagBuilder("f").disablePreprocessing(true).build(); + FeatureFlag marked = source.markedAsOverride(); + assertNull(marked.preprocessed); + assertNull(source.preprocessed); + assertTrue(marked.isOverride()); + } + + @Test + public void markedSegmentCopyCarriesMarkerAndSharesDataWithoutMutatingSource() { + Segment source = segmentBuilder("s").version(3).included("u1").excluded("u2").unbounded(false).build(); + + Segment marked = source.markedAsOverride(); + + assertTrue(marked.isOverride()); + assertFalse(source.isOverride()); + assertNotSame(source, marked); + assertEquals(source.getKey(), marked.getKey()); + assertEquals(source.getVersion(), marked.getVersion()); + assertSame(source.getIncluded(), marked.getIncluded()); + assertSame(source.getExcluded(), marked.getExcluded()); + assertSame(source.getRules(), marked.getRules()); + } + + @Test + public void overrideMarkerIsNeverSerialized() { + FeatureFlag flag = flagBuilder("f").version(7).build().markedAsOverride(); + String flagJson = DataModel.FEATURES.serialize(new ItemDescriptor(flag.getVersion(), flag)); + assertFalse(flagJson.toLowerCase().contains("override")); + assertEquals(LDValue.of("f"), LDValue.parse(flagJson).get("key")); + + Segment segment = segmentBuilder("s").version(3).build().markedAsOverride(); + String segmentJson = DataModel.SEGMENTS.serialize(new ItemDescriptor(segment.getVersion(), segment)); + assertFalse(segmentJson.toLowerCase().contains("override")); + } + + @Test + public void overrideMarkerInJsonIsIgnoredWhenDeserializing() { + FeatureFlag flag = (FeatureFlag) DataModel.FEATURES.deserialize( + "{\"key\":\"f\",\"version\":1,\"isOverride\":true}").getItem(); + assertFalse(flag.isOverride()); + Segment segment = (Segment) DataModel.SEGMENTS.deserialize( + "{\"key\":\"s\",\"version\":1,\"isOverride\":true}").getItem(); + assertFalse(segment.isOverride()); + } } diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataSourceUpdatesImplTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataSourceUpdatesImplTest.java index fee4df7d..59679f60 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataSourceUpdatesImplTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataSourceUpdatesImplTest.java @@ -41,6 +41,7 @@ import static com.launchdarkly.sdk.server.DataModel.FEATURES; import static com.launchdarkly.sdk.server.DataModel.SEGMENTS; +import static com.launchdarkly.sdk.server.ModelBuilders.clauseMatchingSegment; import static com.launchdarkly.sdk.server.ModelBuilders.flagBuilder; import static com.launchdarkly.sdk.server.ModelBuilders.prerequisite; import static com.launchdarkly.sdk.server.ModelBuilders.ruleBuilder; @@ -1053,4 +1054,33 @@ public void applyFullChangeSetToLegacyStoreWithEnvironmentId() throws Exception updates.apply(new ChangeSet<>(ChangeSetType.Full, Selector.make(2, "state2"), data, null, true)); assertNull(legacyStore.getEnvironmentId()); } + + @Test + public void sendsEventsToOverridesThatDependOnAnUpdatedItem() throws Exception { + // The override "overridden" matches the LaunchDarkly segment "segment1". The LaunchDarkly flag + // "parent" has "overridden" as a prerequisite, so a change to the segment reaches parent through + // the override entry: LaunchDarkly data -> override entry -> LaunchDarkly data. + DataStore store = inMemoryDataStore(); + DataSourceUpdatesImpl storeUpdates = makeInstance(store); + OverrideLayer layer = new OverrideLayer(); + layer.setAll(OverrideLayerTest.flagsOnly(flagBuilder("overridden").version(1).on(true) + .rules(ruleBuilder().id("r").variation(0).clauses(clauseMatchingSegment("segment1")).build()).build())); + storeUpdates.setExternalDependencies(layer); + DataBuilder builder = new DataBuilder() + .addAny(FEATURES, + flagBuilder("parent").version(1).prerequisites(prerequisite("overridden", 0)).build()) + .addAny(SEGMENTS, + segmentBuilder("segment1").version(1).build()); + storeUpdates.init(builder.build()); + + BlockingQueue eventSink = new LinkedBlockingQueue<>(); + flagChangeBroadcaster.register(eventSink::add); + + storeUpdates.upsert(SEGMENTS, "segment1", new ItemDescriptor(2, segmentBuilder("segment1").version(2).build())); + expectEvents(eventSink, "overridden", "parent"); + + // An item nothing references notifies no flag. + storeUpdates.upsert(SEGMENTS, "other", new ItemDescriptor(1, segmentBuilder("other").version(1).build())); + assertNull(eventSink.poll(200, TimeUnit.MILLISECONDS)); + } } diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/EvalResultTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/EvalResultTest.java index 71ef5dec..59f1dfd0 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/EvalResultTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/EvalResultTest.java @@ -13,6 +13,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.sameInstance; @SuppressWarnings("javadoc") @@ -134,6 +135,52 @@ public void withForceReasonTracking() { assertThat(r1.getAnyType(), sameInstance(r.getAnyType())); } + @Test + public void overrideAffectedFollowsTheReason() { + EvalResult r = EvalResult.of(SOME_VALUE, SOME_VARIATION, SOME_REASON); + assertThat(r.isOverrideAffected(), is(false)); + assertThat(EvalResult.of(SOME_VALUE, SOME_VARIATION, SOME_REASON.withOverrideAffected(true)).isOverrideAffected(), + is(true)); + assertThat(EvalResult.error(EvaluationReason.ErrorKind.MALFORMED_FLAG).isOverrideAffected(), is(false)); + } + + @Test + public void withOverrideAffected() { + EvalResult r = EvalResult.of(SOME_VALUE, SOME_VARIATION, SOME_REASON); + + // Unchanged value keeps the same instance, so shared precomputed results stay shared. + assertThat(r.withOverrideAffected(false), sameInstance(r)); + + EvalResult marked = r.withOverrideAffected(true); + assertThat(marked, not(sameInstance(r))); + assertThat(marked.isOverrideAffected(), is(true)); + assertThat(marked.getReason(), equalTo(SOME_REASON.withOverrideAffected(true))); + assertThat(marked.getValue(), equalTo(r.getValue())); + assertThat(marked.getVariationIndex(), equalTo(r.getVariationIndex())); + assertThat(marked.withOverrideAffected(true), sameInstance(marked)); + assertThat(marked.withOverrideAffected(false), equalTo(r)); + + // Every typed view carries the marked reason. + assertThat(marked.getAsBoolean().getReason().isOverrideAffected(), is(true)); + assertThat(marked.getAsInteger().getReason().isOverrideAffected(), is(true)); + assertThat(marked.getAsDouble().getReason().isOverrideAffected(), is(true)); + assertThat(marked.getAsString().getReason().isOverrideAffected(), is(true)); + assertThat(marked.getAnyType().getReason().isOverrideAffected(), is(true)); + + // The original is untouched. + assertThat(r.isOverrideAffected(), is(false)); + } + + @Test + public void withOverrideAffectedKeepsPrerequisiteRecordsAndForceTracking() { + EvalResult r = EvalResult.of(SOME_VALUE, SOME_VARIATION, EvaluationReason.fallthrough(true)) + .withPrerequisiteEvalRecords(java.util.Collections.singletonList( + new PrerequisiteEvalRecord(null, null, EvalResult.of(SOME_VALUE, SOME_VARIATION, SOME_REASON)))); + EvalResult marked = r.withOverrideAffected(true); + assertThat(marked.isForceReasonTracking(), is(true)); + assertThat(marked.getPrerequisiteEvalRecords(), sameInstance(r.getPrerequisiteEvalRecords())); + } + private void testForType(T value, LDValue ldValue, Function getter) { assertThat( getter.apply(EvalResult.of(EvaluationDetail.fromValue(ldValue, SOME_VARIATION, SOME_REASON))), diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/EvaluatorOverrideMarkingTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/EvaluatorOverrideMarkingTest.java new file mode 100644 index 00000000..98affee2 --- /dev/null +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/EvaluatorOverrideMarkingTest.java @@ -0,0 +1,346 @@ +package com.launchdarkly.sdk.server; + +import com.launchdarkly.sdk.EvaluationReason; +import com.launchdarkly.sdk.EvaluationReason.ErrorKind; +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.server.DataModel.FeatureFlag; +import com.launchdarkly.sdk.server.DataModel.Segment; +import com.launchdarkly.sdk.server.EvaluatorTestUtil.EvaluatorBuilder; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static com.launchdarkly.sdk.server.EvaluatorTestUtil.BASE_USER; +import static com.launchdarkly.sdk.server.EvaluatorTestUtil.FALLTHROUGH_VALUE; +import static com.launchdarkly.sdk.server.EvaluatorTestUtil.FALLTHROUGH_VARIATION; +import static com.launchdarkly.sdk.server.EvaluatorTestUtil.GREEN_VARIATION; +import static com.launchdarkly.sdk.server.EvaluatorTestUtil.MATCH_VALUE; +import static com.launchdarkly.sdk.server.EvaluatorTestUtil.MATCH_VARIATION; +import static com.launchdarkly.sdk.server.EvaluatorTestUtil.OFF_VALUE; +import static com.launchdarkly.sdk.server.EvaluatorTestUtil.OFF_VARIATION; +import static com.launchdarkly.sdk.server.EvaluatorTestUtil.buildRedGreenFlag; +import static com.launchdarkly.sdk.server.EvaluatorTestUtil.buildThreeWayFlag; +import static com.launchdarkly.sdk.server.EvaluatorTestUtil.evaluatorBuilder; +import static com.launchdarkly.sdk.server.ModelBuilders.clause; +import static com.launchdarkly.sdk.server.ModelBuilders.clauseMatchingSegment; +import static com.launchdarkly.sdk.server.ModelBuilders.flagBuilder; +import static com.launchdarkly.sdk.server.ModelBuilders.negateClause; +import static com.launchdarkly.sdk.server.ModelBuilders.prerequisite; +import static com.launchdarkly.sdk.server.ModelBuilders.ruleBuilder; +import static com.launchdarkly.sdk.server.ModelBuilders.segmentBuilder; +import static com.launchdarkly.sdk.server.ModelBuilders.segmentRuleBuilder; +import static com.launchdarkly.sdk.server.ModelBuilders.target; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * An evaluation is override-affected when any definition it read carries the override marker: the + * evaluated flag, a prerequisite at any depth, or a segment consulted during matching. The marking + * propagates upward only. A prerequisite's own record reflects only its own subtree. + */ +@SuppressWarnings("javadoc") +public class EvaluatorOverrideMarkingTest { + private static final LDContext OTHER_USER = LDContext.create("other"); + + private static final class RecordingRecorder implements EvaluationRecorder { + final List prerequisites = new ArrayList<>(); + + @Override + public void recordPrerequisiteEvaluation(FeatureFlag flag, FeatureFlag prereqOfFlag, LDContext context, EvalResult result) { + prerequisites.add(new PrerequisiteEvalRecord(flag, prereqOfFlag, result)); + } + } + + private static PrerequisiteEvalRecord recordFor(List records, String flagKey) { + for (PrerequisiteEvalRecord r : records) { + if (r.flag.getKey().equals(flagKey)) { + return r; + } + } + throw new AssertionError("no prerequisite record for " + flagKey); + } + + @Test + public void evaluationOfUnmarkedFlagIsNotOverrideAffected() { + FeatureFlag f = buildThreeWayFlag("feature").on(true).build(); + EvalResult result = evaluatorBuilder().build().evaluate(f, BASE_USER, new EvaluationRecorder() {}); + + assertFalse(result.isOverrideAffected()); + assertFalse(result.getReason().isOverrideAffected()); + assertEquals(EvaluationReason.fallthrough(), result.getReason()); + } + + @Test + public void evaluationOfOverrideFlagIsMarkedForEveryReasonKind() { + Evaluator e = evaluatorBuilder().build(); + + FeatureFlag off = buildThreeWayFlag("off").on(false).build().markedAsOverride(); + EvalResult offResult = e.evaluate(off, BASE_USER, new EvaluationRecorder() {}); + assertEquals(EvalResult.of(OFF_VALUE, OFF_VARIATION, EvaluationReason.off().withOverrideAffected(true)), offResult); + assertTrue(offResult.isOverrideAffected()); + + FeatureFlag fallthrough = buildThreeWayFlag("fallthrough").on(true).build().markedAsOverride(); + assertEquals(EvaluationReason.fallthrough().withOverrideAffected(true), + e.evaluate(fallthrough, BASE_USER, new EvaluationRecorder() {}).getReason()); + + FeatureFlag targeted = buildThreeWayFlag("target").on(true) + .targets(target(MATCH_VARIATION, BASE_USER.getKey())).build().markedAsOverride(); + assertEquals(EvaluationReason.targetMatch().withOverrideAffected(true), + e.evaluate(targeted, BASE_USER, new EvaluationRecorder() {}).getReason()); + + FeatureFlag ruled = buildThreeWayFlag("rule").on(true) + .rules(ruleBuilder().id("r").variation(MATCH_VARIATION) + .clauses(clause("key", DataModel.Operator.in, LDValue.of(BASE_USER.getKey()))).build()) + .build().markedAsOverride(); + EvalResult ruleResult = e.evaluate(ruled, BASE_USER, new EvaluationRecorder() {}); + assertEquals(EvaluationReason.ruleMatch(0, "r").withOverrideAffected(true), ruleResult.getReason()); + assertEquals(MATCH_VALUE, ruleResult.getValue()); + } + + @Test + public void evaluationErrorOfOverrideFlagIsMarked() { + // The fallthrough points at a variation that does not exist, which is a malformed flag. + FeatureFlag malformed = flagBuilder("malformed").on(true).variations(LDValue.of("only")) + .fallthroughVariation(5).offVariation(0).build().markedAsOverride(); + EvalResult result = evaluatorBuilder().build().evaluate(malformed, BASE_USER, new EvaluationRecorder() {}); + + assertEquals(EvaluationReason.error(ErrorKind.MALFORMED_FLAG).withOverrideAffected(true), result.getReason()); + assertTrue(result.isNoVariation()); + assertTrue(result.isOverrideAffected()); + } + + @Test + public void thrownEvaluationErrorOfOverrideFlagIsMarked() { + // A clause without an attribute makes the evaluator throw, which becomes an error result. + FeatureFlag broken = buildThreeWayFlag("broken").on(true) + .rules(ruleBuilder().id("r").variation(MATCH_VARIATION) + .clauses(clause(null, DataModel.Operator.in, LDValue.of("x"))).build()) + .build().markedAsOverride(); + EvalResult result = evaluatorBuilder().build().evaluate(broken, BASE_USER, new EvaluationRecorder() {}); + + assertEquals(EvaluationReason.error(ErrorKind.MALFORMED_FLAG).withOverrideAffected(true), result.getReason()); + } + + @Test + public void overridePrerequisiteMarksParentAndItsOwnRecord() { + FeatureFlag parent = buildThreeWayFlag("parent").on(true) + .prerequisites(prerequisite("prereq", GREEN_VARIATION)).build(); + FeatureFlag prereq = buildRedGreenFlag("prereq").on(true).build().markedAsOverride(); + RecordingRecorder recorder = new RecordingRecorder(); + + EvalResult result = evaluatorBuilder().withStoredFlags(prereq).build().evaluate(parent, BASE_USER, recorder); + + assertEquals(FALLTHROUGH_VALUE, result.getValue()); + assertEquals(EvaluationReason.fallthrough().withOverrideAffected(true), result.getReason()); + assertTrue(result.isOverrideAffected()); + + PrerequisiteEvalRecord record = recordFor(recorder.prerequisites, "prereq"); + assertTrue(record.result.isOverrideAffected()); + assertEquals(EvaluationReason.fallthrough().withOverrideAffected(true), record.result.getReason()); + // The same record is on the result. + assertTrue(recordFor(result.getPrerequisiteEvalRecords(), "prereq").result.isOverrideAffected()); + } + + @Test + public void failedOverridePrerequisiteStillMarksParent() { + FeatureFlag parent = buildThreeWayFlag("parent").on(true) + .prerequisites(prerequisite("prereq", GREEN_VARIATION)).build(); + FeatureFlag prereq = buildRedGreenFlag("prereq").on(false).build().markedAsOverride(); + + EvalResult result = evaluatorBuilder().withStoredFlags(prereq).build() + .evaluate(parent, BASE_USER, new EvaluationRecorder() {}); + + assertEquals(OFF_VALUE, result.getValue()); + assertEquals(EvaluationReason.prerequisiteFailed("prereq").withOverrideAffected(true), result.getReason()); + } + + @Test + public void unaffectedPrerequisiteRecordIsNotMarkedInsideMarkedEvaluation() { + FeatureFlag parent = buildThreeWayFlag("parent").on(true) + .prerequisites(prerequisite("overridden", GREEN_VARIATION), prerequisite("plain", GREEN_VARIATION)).build(); + FeatureFlag overridden = buildRedGreenFlag("overridden").on(true).build().markedAsOverride(); + FeatureFlag plain = buildRedGreenFlag("plain").on(true).build(); + RecordingRecorder recorder = new RecordingRecorder(); + + EvalResult result = evaluatorBuilder().withStoredFlags(overridden, plain).build().evaluate(parent, BASE_USER, recorder); + + assertTrue(result.isOverrideAffected()); + assertTrue(recordFor(recorder.prerequisites, "overridden").result.isOverrideAffected()); + assertFalse(recordFor(recorder.prerequisites, "plain").result.isOverrideAffected()); + assertEquals(EvaluationReason.fallthrough(), recordFor(recorder.prerequisites, "plain").result.getReason()); + } + + @Test + public void markingPropagatesUpwardThroughEveryDepthButNotSideways() { + // A depends on B and C. B depends on D, which is the only override. A, B, and D are marked. + // C is not. + FeatureFlag a = buildThreeWayFlag("a").on(true) + .prerequisites(prerequisite("b", GREEN_VARIATION), prerequisite("c", GREEN_VARIATION)).build(); + FeatureFlag b = buildRedGreenFlag("b").on(true).prerequisites(prerequisite("d", GREEN_VARIATION)).build(); + FeatureFlag c = buildRedGreenFlag("c").on(true).build(); + FeatureFlag d = buildRedGreenFlag("d").on(true).build().markedAsOverride(); + RecordingRecorder recorder = new RecordingRecorder(); + + EvalResult result = evaluatorBuilder().withStoredFlags(b, c, d).build().evaluate(a, BASE_USER, recorder); + + assertTrue(result.isOverrideAffected()); + assertTrue(recordFor(recorder.prerequisites, "b").result.isOverrideAffected()); + assertTrue(recordFor(recorder.prerequisites, "d").result.isOverrideAffected()); + assertFalse(recordFor(recorder.prerequisites, "c").result.isOverrideAffected()); + } + + @Test + public void overrideFlagWithUnmarkedPrerequisiteMarksOnlyItself() { + FeatureFlag parent = buildThreeWayFlag("parent").on(true) + .prerequisites(prerequisite("prereq", GREEN_VARIATION)).build().markedAsOverride(); + FeatureFlag prereq = buildRedGreenFlag("prereq").on(true).build(); + RecordingRecorder recorder = new RecordingRecorder(); + + EvalResult result = evaluatorBuilder().withStoredFlags(prereq).build().evaluate(parent, BASE_USER, recorder); + + assertTrue(result.isOverrideAffected()); + // The parent's marking does not flow down into the prerequisite's record. + assertFalse(recordFor(recorder.prerequisites, "prereq").result.isOverrideAffected()); + } + + @Test + public void circularReferenceThroughOverridePrerequisiteIsMarkedError() { + FeatureFlag a = buildThreeWayFlag("a").on(true).prerequisites(prerequisite("b", GREEN_VARIATION)).build(); + FeatureFlag b = buildRedGreenFlag("b").on(true).prerequisites(prerequisite("a", GREEN_VARIATION)).build() + .markedAsOverride(); + + EvalResult result = evaluatorBuilder().withStoredFlags(b).build().evaluate(a, BASE_USER, new EvaluationRecorder() {}); + + assertEquals(EvaluationReason.error(ErrorKind.MALFORMED_FLAG).withOverrideAffected(true), result.getReason()); + } + + @Test + public void matchingOverrideSegmentMarksEvaluation() { + Segment segment = segmentBuilder("seg").included(BASE_USER.getKey()).build().markedAsOverride(); + FeatureFlag f = buildThreeWayFlag("feature").on(true) + .rules(ruleBuilder().id("r").variation(MATCH_VARIATION).clauses(clauseMatchingSegment("seg")).build()) + .build(); + + EvalResult result = evaluatorBuilder().withStoredSegments(segment).build() + .evaluate(f, BASE_USER, new EvaluationRecorder() {}); + + assertEquals(MATCH_VALUE, result.getValue()); + assertEquals(EvaluationReason.ruleMatch(0, "r").withOverrideAffected(true), result.getReason()); + } + + @Test + public void nonMatchingOverrideSegmentStillMarksEvaluation() { + Segment segment = segmentBuilder("seg").included(BASE_USER.getKey()).build().markedAsOverride(); + FeatureFlag f = buildThreeWayFlag("feature").on(true) + .rules(ruleBuilder().id("r").variation(MATCH_VARIATION).clauses(clauseMatchingSegment("seg")).build()) + .build(); + + EvalResult result = evaluatorBuilder().withStoredSegments(segment).build() + .evaluate(f, OTHER_USER, new EvaluationRecorder() {}); + + assertEquals(FALLTHROUGH_VALUE, result.getValue()); + assertEquals(EvaluationReason.fallthrough().withOverrideAffected(true), result.getReason()); + } + + @Test + public void negatedClauseOnOverrideSegmentMarksEvaluation() { + Segment segment = segmentBuilder("seg").included(BASE_USER.getKey()).build().markedAsOverride(); + FeatureFlag f = buildThreeWayFlag("feature").on(true) + .rules(ruleBuilder().id("r").variation(MATCH_VARIATION) + .clauses(negateClause(clauseMatchingSegment("seg"))).build()) + .build(); + + EvalResult result = evaluatorBuilder().withStoredSegments(segment).build() + .evaluate(f, OTHER_USER, new EvaluationRecorder() {}); + + assertEquals(MATCH_VALUE, result.getValue()); + assertTrue(result.isOverrideAffected()); + } + + @Test + public void unmarkedSegmentDoesNotMarkEvaluation() { + Segment segment = segmentBuilder("seg").included(BASE_USER.getKey()).build(); + FeatureFlag f = buildThreeWayFlag("feature").on(true) + .rules(ruleBuilder().id("r").variation(MATCH_VARIATION).clauses(clauseMatchingSegment("seg")).build()) + .build(); + + EvalResult result = evaluatorBuilder().withStoredSegments(segment).build() + .evaluate(f, BASE_USER, new EvaluationRecorder() {}); + + assertEquals(MATCH_VALUE, result.getValue()); + assertFalse(result.isOverrideAffected()); + } + + @Test + public void overrideSegmentReferencedByAnotherSegmentMarksEvaluation() { + Segment inner = segmentBuilder("inner").included(BASE_USER.getKey()).build().markedAsOverride(); + Segment outer = segmentBuilder("outer") + .rules(segmentRuleBuilder().clauses(clauseMatchingSegment("inner")).build()).build(); + FeatureFlag f = buildThreeWayFlag("feature").on(true) + .rules(ruleBuilder().id("r").variation(MATCH_VARIATION).clauses(clauseMatchingSegment("outer")).build()) + .build(); + + EvalResult result = evaluatorBuilder().withStoredSegments(inner, outer).build() + .evaluate(f, BASE_USER, new EvaluationRecorder() {}); + + assertEquals(MATCH_VALUE, result.getValue()); + assertTrue(result.isOverrideAffected()); + } + + @Test + public void segmentReadByPrerequisiteMarksPrerequisiteAndParent() { + Segment segment = segmentBuilder("seg").included(BASE_USER.getKey()).build().markedAsOverride(); + FeatureFlag parent = buildThreeWayFlag("parent").on(true) + .prerequisites(prerequisite("prereq", GREEN_VARIATION)).build(); + FeatureFlag prereq = buildRedGreenFlag("prereq").on(true) + .rules(ruleBuilder().id("r").variation(GREEN_VARIATION).clauses(clauseMatchingSegment("seg")).build()) + .build(); + RecordingRecorder recorder = new RecordingRecorder(); + + EvalResult result = evaluatorBuilder().withStoredFlags(prereq).withStoredSegments(segment).build() + .evaluate(parent, BASE_USER, recorder); + + assertTrue(result.isOverrideAffected()); + assertTrue(recordFor(recorder.prerequisites, "prereq").result.isOverrideAffected()); + } + + @Test + public void markedEvaluationDoesNotAlterSharedPrecomputedResults() { + FeatureFlag source = buildThreeWayFlag("feature").on(false).build(); + FeatureFlag marked = source.markedAsOverride(); + Evaluator e = evaluatorBuilder().build(); + + EvalResult markedResult = e.evaluate(marked, BASE_USER, new EvaluationRecorder() {}); + EvalResult sourceResult = e.evaluate(source, BASE_USER, new EvaluationRecorder() {}); + + assertTrue(markedResult.isOverrideAffected()); + assertFalse(sourceResult.isOverrideAffected()); + assertFalse(source.isOverride()); + assertNotSame(markedResult, sourceResult); + // The unmarked evaluation still returns the shared precomputed instance. + assertSame(sourceResult, e.evaluate(source, BASE_USER, new EvaluationRecorder() {})); + } + + @Test + public void bigSegmentsStatusAndOverrideMarkingAreBothKept() { + Segment bigSegment = segmentBuilder("big").unbounded(true).generation(1).build().markedAsOverride(); + FeatureFlag f = buildThreeWayFlag("feature").on(true) + .rules(ruleBuilder().id("r").variation(MATCH_VARIATION).clauses(clauseMatchingSegment("big")).build()) + .build(); + // No big segment store is configured, so the status is NOT_CONFIGURED. + EvaluatorBuilder builder = evaluatorBuilder().withStoredSegments(bigSegment) + .withBigSegmentQueryResult(BASE_USER.getKey(), null); + + EvalResult result = builder.build().evaluate(f, BASE_USER, new EvaluationRecorder() {}); + + assertEquals(FALLTHROUGH_VARIATION, result.getVariationIndex()); + assertEquals(EvaluationReason.BigSegmentsStatus.NOT_CONFIGURED, result.getReason().getBigSegmentsStatus()); + assertTrue(result.getReason().isOverrideAffected()); + } +} diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/InputValidatingEvaluatorOverrideTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/InputValidatingEvaluatorOverrideTest.java new file mode 100644 index 00000000..b8442ee3 --- /dev/null +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/InputValidatingEvaluatorOverrideTest.java @@ -0,0 +1,206 @@ +package com.launchdarkly.sdk.server; + +import com.launchdarkly.sdk.EvaluationReason; +import com.launchdarkly.sdk.EvaluationReason.ErrorKind; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.LDValueType; +import com.launchdarkly.sdk.internal.events.Event; +import com.launchdarkly.sdk.json.JsonSerialization; +import com.launchdarkly.sdk.server.DataModel.FeatureFlag; +import com.launchdarkly.sdk.server.TestComponents.TestEventProcessor; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.DataKind; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.KeyedItems; + +import org.junit.Test; + +import java.util.AbstractMap; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static com.launchdarkly.sdk.server.DataModel.FEATURES; +import static com.launchdarkly.sdk.server.EvaluatorTestUtil.BASE_USER; +import static com.launchdarkly.sdk.server.ModelBuilders.flagBuilder; +import static com.launchdarkly.sdk.server.ModelBuilders.prerequisite; +import static com.launchdarkly.sdk.server.TestComponents.nullLogger; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +@SuppressWarnings("javadoc") +public class InputValidatingEvaluatorOverrideTest { + private static ReadOnlyStore storeWith(FeatureFlag... flags) { + Map items = new HashMap<>(); + for (FeatureFlag f : flags) { + items.put(f.getKey(), new ItemDescriptor(f.getVersion(), f)); + } + return new ReadOnlyStore() { + @Override + public ItemDescriptor get(DataKind kind, String key) { + return kind == FEATURES ? items.get(key) : null; + } + + @Override + public KeyedItems getAll(DataKind kind) { + return kind == FEATURES ? new KeyedItems<>(items.entrySet()) : new KeyedItems<>(Collections.emptyList()); + } + + @Override + public boolean isInitialized() { + return true; + } + }; + } + + // A store that holds the given flags and fails when the given key is read, as a store does + // when its database is unreachable. + private static ReadOnlyStore storeFailingOn(String failingKey, FeatureFlag... flags) { + ReadOnlyStore base = storeWith(flags); + return new ReadOnlyStore() { + @Override + public ItemDescriptor get(DataKind kind, String key) { + if (key.equals(failingKey)) { + throw new RuntimeException("store failed"); + } + return base.get(kind, key); + } + + @Override + public KeyedItems getAll(DataKind kind) { + return base.getAll(kind); + } + + @Override + public boolean isInitialized() { + return true; + } + }; + } + + // A flag whose evaluation reads the prerequisite "prereq". + private static FeatureFlag flagWithPrerequisite(String key) { + return flagBuilder(key).on(true).prerequisites(prerequisite("prereq", 0)).fallthroughVariation(0) + .offVariation(0).variations(LDValue.of("value")).trackEvents(true).build(); + } + + @Test + public void exceptionWhileEvaluatingOverrideFlagKeepsMarking() { + FeatureFlag flag = flagWithPrerequisite("flag").markedAsOverride(); + EvalResultAndFlag result = evaluatorOver(storeFailingOn("prereq", flag)).evaluate("flag", BASE_USER, + LDValue.of("default"), null, InputValidatingEvaluator.NO_OP_EVALUATION_EVENT_RECORDER); + + assertEquals(LDValue.of("default"), result.getResult().getValue()); + assertEquals(EvaluationReason.ErrorKind.EXCEPTION, result.getResult().getReason().getErrorKind()); + assertTrue(result.getResult().getReason().isOverrideAffected()); + assertTrue(result.getResult().isOverrideAffected()); + } + + @Test + public void exceptionWhileEvaluatingOverrideFlagRecordsAMarkedErrorEvent() { + TestEventProcessor events = new TestEventProcessor(); + FeatureFlag flag = flagWithPrerequisite("flag").markedAsOverride(); + new InputValidatingEvaluator(storeFailingOn("prereq", flag), null, null, events, nullLogger) + .evalAndFlag("LDClient.stringVariation", "flag", BASE_USER, LDValue.of("default"), LDValueType.STRING, + EvaluationOptions.EVENTS_WITH_REASONS); + + assertEquals(1, events.events.size()); + Event.FeatureRequest record = (Event.FeatureRequest) events.events.get(0); + assertTrue(record.isOverrideAffected()); + assertEquals(EvaluationReason.ErrorKind.EXCEPTION, record.getReason().getErrorKind()); + assertTrue(record.getReason().isOverrideAffected()); + } + + @Test + public void exceptionWhileEvaluatingPlainFlagIsNotMarked() { + FeatureFlag flag = flagWithPrerequisite("flag"); + EvalResultAndFlag result = evaluatorOver(storeFailingOn("prereq", flag)).evaluate("flag", BASE_USER, + LDValue.of("default"), null, InputValidatingEvaluator.NO_OP_EVALUATION_EVENT_RECORDER); + + assertEquals(EvaluationReason.ErrorKind.EXCEPTION, result.getResult().getReason().getErrorKind()); + assertFalse(result.getResult().isOverrideAffected()); + } + + private static InputValidatingEvaluator evaluatorOver(ReadOnlyStore store) { + return new InputValidatingEvaluator(store, null, null, new NoOpEventProcessor(), nullLogger); + } + + @Test + public void typeMismatchOnOverrideFlagKeepsMarking() { + FeatureFlag flag = flagBuilder("flag").on(false).offVariation(0).variations(LDValue.of("a string")) + .build().markedAsOverride(); + EvalResultAndFlag result = evaluatorOver(storeWith(flag)).evaluate("flag", BASE_USER, LDValue.of(true), + LDValueType.BOOLEAN, InputValidatingEvaluator.NO_OP_EVALUATION_EVENT_RECORDER); + + assertEquals(LDValue.of(true), result.getResult().getValue()); + assertEquals(EvaluationReason.error(ErrorKind.WRONG_TYPE).withOverrideAffected(true), result.getResult().getReason()); + assertTrue(result.getResult().isOverrideAffected()); + } + + @Test + public void unrecognizedMigrationStageFromOverrideFlagKeepsMarking() { + FeatureFlag flag = flagBuilder("flag").on(false).offVariation(0).variations(LDValue.of("potato")) + .build().markedAsOverride(); + MigrationStageEnforcingEvaluator evaluator = + new MigrationStageEnforcingEvaluator(evaluatorOver(storeWith(flag)), nullLogger); + EvalResultAndFlag result = evaluator.evalAndFlag("LDClient.migrationVariation", "flag", BASE_USER, + LDValue.of("off"), LDValueType.STRING, EvaluationOptions.EVENTS_WITHOUT_REASONS); + + assertEquals(LDValue.of("off"), result.getResult().getValue()); + assertEquals(EvaluationReason.error(ErrorKind.WRONG_TYPE).withOverrideAffected(true), result.getResult().getReason()); + } + + @Test + public void typeMismatchOnOrdinaryFlagIsNotMarked() { + FeatureFlag flag = flagBuilder("flag").on(false).offVariation(0).variations(LDValue.of("a string")).build(); + EvalResultAndFlag result = evaluatorOver(storeWith(flag)).evaluate("flag", BASE_USER, LDValue.of(true), + LDValueType.BOOLEAN, InputValidatingEvaluator.NO_OP_EVALUATION_EVENT_RECORDER); + + assertEquals(EvaluationReason.error(ErrorKind.WRONG_TYPE), result.getResult().getReason()); + assertFalse(result.getResult().isOverrideAffected()); + } + + @Test + public void errorResultOfOverrideFlagKeepsMarkingWithCallerDefault() { + FeatureFlag malformed = flagBuilder("flag").on(true).variations(LDValue.of("only")) + .fallthroughVariation(5).offVariation(0).build().markedAsOverride(); + EvalResultAndFlag result = evaluatorOver(storeWith(malformed)).evaluate("flag", BASE_USER, LDValue.of("fallback"), + null, InputValidatingEvaluator.NO_OP_EVALUATION_EVENT_RECORDER); + + assertEquals(LDValue.of("fallback"), result.getResult().getValue()); + assertTrue(result.getResult().isNoVariation()); + assertEquals(EvaluationReason.error(ErrorKind.MALFORMED_FLAG).withOverrideAffected(true), result.getResult().getReason()); + } + + @Test + public void allFlagsStateMarksTheExceptionResultOfAnOverrideFlagAndTurnsOffItsTracking() { + FeatureFlag marked = flagWithPrerequisite("marked").markedAsOverride(); + FeatureFlag plain = flagWithPrerequisite("plain"); + FeatureFlagsState state = evaluatorOver(storeFailingOn("prereq", marked, plain)) + .allFlagsState(BASE_USER, FlagsStateOption.WITH_REASONS); + LDValue flagsState = LDValue.parse(JsonSerialization.serialize(state)).get("$flagsState"); + + assertEquals(EvaluationReason.ErrorKind.EXCEPTION, state.getFlagReason("marked").getErrorKind()); + assertTrue(state.getFlagReason("marked").isOverrideAffected()); + assertTrue(flagsState.get("marked").get("trackEvents").isNull()); + + // A flag that did not come from the override store keeps its unmarked reason and tracking. + assertEquals(EvaluationReason.ErrorKind.EXCEPTION, state.getFlagReason("plain").getErrorKind()); + assertFalse(state.getFlagReason("plain").isOverrideAffected()); + assertEquals(LDValue.of(true), flagsState.get("plain").get("trackEvents")); + } + + @Test + public void allFlagsStateKeepsMarkedReason() { + FeatureFlag marked = flagBuilder("marked").on(false).offVariation(0).variations(LDValue.of("x")).build() + .markedAsOverride(); + FeatureFlag plain = flagBuilder("plain").on(false).offVariation(0).variations(LDValue.of("y")).build(); + FeatureFlagsState state = evaluatorOver(storeWith(marked, plain)).allFlagsState(BASE_USER, FlagsStateOption.WITH_REASONS); + + assertTrue(state.isValid()); + assertTrue(state.getFlagReason("marked").isOverrideAffected()); + assertFalse(state.getFlagReason("plain").isOverrideAffected()); + Map.Entry expected = new AbstractMap.SimpleEntry<>("marked", LDValue.of("x")); + assertEquals(expected.getValue(), state.getFlagValue("marked")); + } +} diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/LDClientOverrideEventsTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/LDClientOverrideEventsTest.java new file mode 100644 index 00000000..a56bf9ae --- /dev/null +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/LDClientOverrideEventsTest.java @@ -0,0 +1,271 @@ +package com.launchdarkly.sdk.server; + +import com.launchdarkly.logging.LDLogLevel; +import com.launchdarkly.sdk.EvaluationReason; +import com.launchdarkly.sdk.EvaluationReason.ErrorKind; +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.internal.events.Event; +import com.launchdarkly.sdk.server.DataModel.FeatureFlag; +import com.launchdarkly.sdk.server.DataModel.Segment; +import com.launchdarkly.sdk.server.TestComponents.TestEventProcessor; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.DataKind; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.KeyedItems; +import com.launchdarkly.sdk.server.subsystems.EventSender; + +import org.junit.Test; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import static com.launchdarkly.sdk.server.ModelBuilders.clauseMatchingSegment; +import static com.launchdarkly.sdk.server.ModelBuilders.flagBuilder; +import static com.launchdarkly.sdk.server.ModelBuilders.prerequisite; +import static com.launchdarkly.sdk.server.ModelBuilders.ruleBuilder; +import static com.launchdarkly.sdk.server.ModelBuilders.segmentBuilder; +import static com.launchdarkly.sdk.server.OverrideLayerTest.collections; +import static com.launchdarkly.sdk.server.OverrideLayerTest.flagsOnly; +import static com.launchdarkly.sdk.server.OverrideTestDataSources.initializerWith; +import static com.launchdarkly.sdk.server.TestComponents.specificComponent; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +/** + * The override-affected marking as the client hands it to the event processor, and the resulting + * analytics output: no individual feature or debug event for a marked evaluation, and a summary + * counter that carries the marker. + */ +@SuppressWarnings("javadoc") +public class LDClientOverrideEventsTest extends BaseTest { + private static final String SDK_KEY = "sdk-key"; + private static final LDContext CONTEXT = LDContext.create("user-key"); + private static final LDValue DEFAULT = LDValue.of("default"); + + private static FeatureFlag valueFlag(String key, int version, String value) { + return flagBuilder(key).version(version).on(false).offVariation(0).variations(LDValue.of(value)).build(); + } + + private LDConfig.Builder baseBuilder(Iterable>> ldData, + TestOverrideSource source) { + return new LDConfig.Builder() + .dataSystem(Components.dataSystem().custom().initializers(initializerWith(ldData)).overrides(source)) + .logging(Components.logging(testLogging).level(LDLogLevel.DEBUG)); + } + + private static List featureRecords(TestEventProcessor events, String flagKey) { + List result = new ArrayList<>(); + for (Event e : events.events) { + if (e instanceof Event.FeatureRequest && ((Event.FeatureRequest) e).getKey().equals(flagKey)) { + result.add((Event.FeatureRequest) e); + } + } + return result; + } + + private static Event.FeatureRequest singleRecord(TestEventProcessor events, String flagKey) { + List records = featureRecords(events, flagKey); + assertEquals("expected one record for " + flagKey, 1, records.size()); + return records.get(0); + } + + @Test + public void evaluationOfOverriddenFlagIsRecordedAsOverrideAffected() throws Exception { + TestEventProcessor events = new TestEventProcessor(); + TestOverrideSource source = new TestOverrideSource(flagsOnly(valueFlag("overridden", 5, "override-value"))); + Iterable>> ldData = flagsOnly( + valueFlag("overridden", 1, "ld-value"), valueFlag("plain", 1, "plain-value")); + + try (LDClient client = new LDClient(SDK_KEY, baseBuilder(ldData, source).events(specificComponent(events)).build())) { + client.stringVariation("overridden", CONTEXT, "default"); + client.stringVariation("plain", CONTEXT, "default"); + + Event.FeatureRequest overridden = singleRecord(events, "overridden"); + assertTrue(overridden.isOverrideAffected()); + assertEquals(5, overridden.getVersion()); + assertEquals(LDValue.of("override-value"), overridden.getValue()); + + Event.FeatureRequest plain = singleRecord(events, "plain"); + assertFalse(plain.isOverrideAffected()); + } + } + + @Test + public void recordedMarkingDoesNotDependOnRequestingTheReason() throws Exception { + TestEventProcessor events = new TestEventProcessor(); + TestOverrideSource source = new TestOverrideSource(flagsOnly(valueFlag("overridden", 5, "override-value"))); + + try (LDClient client = new LDClient(SDK_KEY, baseBuilder(flagsOnly(), source).events(specificComponent(events)).build())) { + client.stringVariation("overridden", CONTEXT, "default"); + Event.FeatureRequest record = singleRecord(events, "overridden"); + assertTrue(record.isOverrideAffected()); + assertNull(record.getReason()); + } + } + + @Test + public void prerequisiteRecordsCarryTheirOwnMarking() throws Exception { + TestEventProcessor events = new TestEventProcessor(); + FeatureFlag parent = flagBuilder("parent").version(1).on(true).fallthroughVariation(1).offVariation(0) + .variations(LDValue.of("off"), LDValue.of("on")) + .prerequisites(prerequisite("overridden-prereq", 1), prerequisite("plain-prereq", 1)).build(); + FeatureFlag ldOverriddenPrereq = flagBuilder("overridden-prereq").version(1).on(false).offVariation(0) + .variations(LDValue.of("a"), LDValue.of("b")).build(); + FeatureFlag plainPrereq = flagBuilder("plain-prereq").version(1).on(true).fallthroughVariation(1).offVariation(0) + .variations(LDValue.of("a"), LDValue.of("b")).build(); + FeatureFlag overriddenPrereq = flagBuilder("overridden-prereq").version(2).on(true).fallthroughVariation(1) + .offVariation(0).variations(LDValue.of("a"), LDValue.of("b")).build(); + TestOverrideSource source = new TestOverrideSource(flagsOnly(overriddenPrereq)); + + try (LDClient client = new LDClient(SDK_KEY, + baseBuilder(flagsOnly(parent, ldOverriddenPrereq, plainPrereq), source).events(specificComponent(events)).build())) { + assertEquals("on", client.stringVariation("parent", CONTEXT, "default")); + + assertTrue(singleRecord(events, "parent").isOverrideAffected()); + Event.FeatureRequest overriddenRecord = singleRecord(events, "overridden-prereq"); + assertTrue(overriddenRecord.isOverrideAffected()); + assertEquals("parent", overriddenRecord.getPrereqOf()); + assertEquals(2, overriddenRecord.getVersion()); + Event.FeatureRequest plainRecord = singleRecord(events, "plain-prereq"); + assertFalse(plainRecord.isOverrideAffected()); + assertEquals("parent", plainRecord.getPrereqOf()); + } + } + + @Test + public void segmentReadMarksTheRecord() throws Exception { + TestEventProcessor events = new TestEventProcessor(); + FeatureFlag usesSegment = flagBuilder("uses-segment").version(1).on(true).fallthroughVariation(0).offVariation(0) + .variations(LDValue.of("no"), LDValue.of("yes")) + .rules(ruleBuilder().id("r").variation(1).clauses(clauseMatchingSegment("seg")).build()).build(); + Segment ldSegment = segmentBuilder("seg").version(1).build(); + Segment overrideSegment = segmentBuilder("seg").version(2).build(); // still does not include the context + TestOverrideSource source = new TestOverrideSource( + collections(Collections.emptyList(), Collections.singletonList(overrideSegment))); + + try (LDClient client = new LDClient(SDK_KEY, baseBuilder( + collections(Collections.singletonList(usesSegment), Collections.singletonList(ldSegment)), source) + .events(specificComponent(events)).build())) { + assertEquals("no", client.stringVariation("uses-segment", CONTEXT, "default")); + assertTrue(singleRecord(events, "uses-segment").isOverrideAffected()); + } + } + + @Test + public void typeMismatchRecordKeepsMarking() throws Exception { + TestEventProcessor events = new TestEventProcessor(); + TestOverrideSource source = new TestOverrideSource(flagsOnly(valueFlag("overridden", 5, "a string"))); + + try (LDClient client = new LDClient(SDK_KEY, baseBuilder(flagsOnly(), source).events(specificComponent(events)).build())) { + assertTrue(client.boolVariation("overridden", CONTEXT, true)); + Event.FeatureRequest record = singleRecord(events, "overridden"); + assertTrue(record.isOverrideAffected()); + assertEquals(EvaluationReason.error(ErrorKind.WRONG_TYPE).withOverrideAffected(true), + client.boolVariationDetail("overridden", CONTEXT, true).getReason()); + } + } + + @Test + public void unknownFlagRecordIsNotMarked() throws Exception { + TestEventProcessor events = new TestEventProcessor(); + TestOverrideSource source = new TestOverrideSource(flagsOnly(valueFlag("overridden", 5, "v"))); + try (LDClient client = new LDClient(SDK_KEY, baseBuilder(flagsOnly(), source).events(specificComponent(events)).build())) { + client.stringVariation("missing", CONTEXT, "default"); + assertFalse(singleRecord(events, "missing").isOverrideAffected()); + } + } + + /** + * Captures the analytics payloads that the default event processor sends. + */ + private static final class CapturingEventSender implements EventSender { + final BlockingQueue payloads = new LinkedBlockingQueue<>(); + + @Override + public Result sendAnalyticsEvents(byte[] data, int eventCount, URI eventsBaseUri) { + payloads.add(LDValue.parse(new String(data, StandardCharsets.UTF_8))); + return Result.SUCCESS; + } + + @Override + public Result sendDiagnosticEvent(byte[] data, URI eventsBaseUri) { + return Result.SUCCESS; + } + + @Override + public void close() { + } + } + + private static List eventsOfKind(LDValue payload, String kind) { + List result = new ArrayList<>(); + for (LDValue e : payload.values()) { + if (e.get("kind").stringValue().equals(kind)) { + result.add(e); + } + } + return result; + } + + @Test + public void overrideAffectedEvaluationsAppearOnlyInSummaryOutput() throws Exception { + long debugUntil = System.currentTimeMillis() + 100000; + // Both flags request individual feature events and debug events. + FeatureFlag overridden = flagBuilder("overridden").version(300).on(false).offVariation(0) + .variations(LDValue.of("override-value")).trackEvents(true).debugEventsUntilDate(debugUntil).build(); + FeatureFlag plain = flagBuilder("plain").version(100).on(false).offVariation(0) + .variations(LDValue.of("plain-value")).trackEvents(true).debugEventsUntilDate(debugUntil).build(); + TestOverrideSource source = new TestOverrideSource(flagsOnly(overridden)); + CapturingEventSender sender = new CapturingEventSender(); + + LDConfig config = baseBuilder(flagsOnly(plain), source) + .events(Components.sendEvents().eventSender(specificComponent(sender)).flushInterval(Duration.ofHours(1))) + .diagnosticOptOut(true) + .build(); + + try (LDClient client = new LDClient(SDK_KEY, config)) { + client.stringVariation("overridden", CONTEXT, "default"); + client.stringVariation("overridden", CONTEXT, "default"); + client.stringVariation("plain", CONTEXT, "default"); + client.flush(); + + LDValue payload = sender.payloads.poll(5, TimeUnit.SECONDS); + assertNotNull("no analytics payload was sent", payload); + + // The overridden flag produced no feature event and no debug event. The plain flag produced both. + List featureEvents = eventsOfKind(payload, "feature"); + List debugEvents = eventsOfKind(payload, "debug"); + assertEquals(1, featureEvents.size()); + assertEquals("plain", featureEvents.get(0).get("key").stringValue()); + assertEquals(1, debugEvents.size()); + assertEquals("plain", debugEvents.get(0).get("key").stringValue()); + assertEquals(1, eventsOfKind(payload, "index").size()); + + // Both flags are summarized. Only the overridden flag's counter carries the marker. + List summaries = eventsOfKind(payload, "summary"); + assertEquals(1, summaries.size()); + LDValue features = summaries.get(0).get("features"); + LDValue overriddenCounters = features.get("overridden").get("counters"); + assertEquals(1, overriddenCounters.size()); + LDValue overriddenCounter = overriddenCounters.get(0); + assertTrue(overriddenCounter.get("overrideAffected").booleanValue()); + assertEquals(300, overriddenCounter.get("version").intValue()); + assertEquals(2, overriddenCounter.get("count").intValue()); + assertEquals(LDValue.of("override-value"), overriddenCounter.get("value")); + LDValue plainCounter = features.get("plain").get("counters").get(0); + assertTrue(plainCounter.get("overrideAffected").isNull()); + assertEquals(1, plainCounter.get("count").intValue()); + } + } +} diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/LDClientOverridesTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/LDClientOverridesTest.java new file mode 100644 index 00000000..10d64336 --- /dev/null +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/LDClientOverridesTest.java @@ -0,0 +1,652 @@ +package com.launchdarkly.sdk.server; + +import com.launchdarkly.logging.LDLogLevel; +import com.launchdarkly.sdk.EvaluationDetail; +import com.launchdarkly.sdk.EvaluationReason; +import com.launchdarkly.sdk.EvaluationReason.ErrorKind; +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.json.JsonSerialization; +import com.launchdarkly.sdk.server.DataModel.FeatureFlag; +import com.launchdarkly.sdk.server.DataModel.Segment; +import com.launchdarkly.sdk.server.integrations.DataSystemBuilder; +import com.launchdarkly.sdk.server.integrations.FileOverrides; +import com.launchdarkly.sdk.server.integrations.TestDataV2; +import com.launchdarkly.sdk.server.interfaces.FlagChangeEvent; +import com.launchdarkly.sdk.server.interfaces.DataStoreStatusProvider.CacheStats; +import com.launchdarkly.sdk.server.subsystems.ComponentConfigurer; +import com.launchdarkly.sdk.server.subsystems.DataStore; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.DataKind; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.FullDataSet; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.KeyedItems; +import com.launchdarkly.sdk.server.subsystems.DataSystemConfiguration.DataStoreMode; +import com.launchdarkly.sdk.server.subsystems.OverrideSource; +import com.launchdarkly.testhelpers.TempDir; + +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import static com.launchdarkly.sdk.server.ModelBuilders.clauseMatchingSegment; +import static com.launchdarkly.sdk.server.ModelBuilders.flagBuilder; +import static com.launchdarkly.sdk.server.ModelBuilders.prerequisite; +import static com.launchdarkly.sdk.server.ModelBuilders.ruleBuilder; +import static com.launchdarkly.sdk.server.ModelBuilders.segmentBuilder; +import static com.launchdarkly.sdk.server.OverrideLayerTest.collections; +import static com.launchdarkly.sdk.server.OverrideLayerTest.flagsOnly; +import static com.launchdarkly.sdk.server.OverrideTestDataSources.hangingSynchronizer; +import static com.launchdarkly.sdk.server.OverrideTestDataSources.initializerWith; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.not; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * The override layer through the client: the not-initialized short-circuit, the all-flags state, + * flag change notifications, and the override source lifecycle. + */ +@SuppressWarnings("javadoc") +public class LDClientOverridesTest extends BaseTest { + private static final String SDK_KEY = "sdk-key"; + private static final LDContext CONTEXT = LDContext.create("user-key"); + private static final LDValue DEFAULT = LDValue.of("default"); + + private static FeatureFlag valueFlag(String key, String value) { + return flagBuilder(key).version(1).on(true).fallthroughVariation(0).variations(LDValue.of(value)).build(); + } + + private LDConfig.Builder configWith(DataSystemBuilder dataSystem) { + return new LDConfig.Builder() + .dataSystem(dataSystem) + .events(Components.noEvents()) + .logging(Components.logging(testLogging).level(LDLogLevel.DEBUG)); + } + + private LDConfig initializedConfig(Iterable>> ldData, + ComponentConfigurer source) { + return configWith(Components.dataSystem().custom().initializers(initializerWith(ldData)).overrides(source)).build(); + } + + private LDConfig uninitializedConfig(ComponentConfigurer source) { + return configWith(Components.dataSystem().custom().synchronizers(hangingSynchronizer()).overrides(source)) + .startWait(Duration.ZERO) + .build(); + } + + private static void assertOverrideServed(EvaluationDetail detail, String value) { + assertEquals(LDValue.of(value), detail.getValue()); + assertEquals(0, detail.getVariationIndex()); + assertEquals(EvaluationReason.fallthrough().withOverrideAffected(true), detail.getReason()); + } + + private static void assertNotReady(EvaluationDetail detail) { + assertEquals(DEFAULT, detail.getValue()); + assertTrue(detail.isDefaultValue()); + assertEquals(EvaluationReason.error(ErrorKind.CLIENT_NOT_READY), detail.getReason()); + } + + @Test + public void overrideIsServedWhenClientIsNotInitialized() throws Exception { + TestOverrideSource source = new TestOverrideSource(flagsOnly(valueFlag("overridden", "override-value"))); + try (LDClient client = new LDClient(SDK_KEY, uninitializedConfig(source))) { + assertFalse(client.isInitialized()); + assertOverrideServed(client.jsonValueVariationDetail("overridden", CONTEXT, DEFAULT), "override-value"); + assertEquals("override-value", client.stringVariation("overridden", CONTEXT, "default")); + } + } + + @Test + public void nonOverriddenFlagStillShortCircuitsWhenClientIsNotInitialized() throws Exception { + TestOverrideSource source = new TestOverrideSource(flagsOnly(valueFlag("overridden", "override-value"))); + try (LDClient client = new LDClient(SDK_KEY, uninitializedConfig(source))) { + assertNotReady(client.jsonValueVariationDetail("other", CONTEXT, DEFAULT)); + assertThat(logCapture.getMessageStrings(), hasItem( + "WARN:Evaluation called before client initialized for feature flag \"other\"; data store unavailable, returning default value")); + } + } + + @Test + public void overrideRemovalRestoresShortCircuit() throws Exception { + TestOverrideSource source = new TestOverrideSource(flagsOnly(valueFlag("overridden", "override-value"))); + try (LDClient client = new LDClient(SDK_KEY, uninitializedConfig(source))) { + assertOverrideServed(client.jsonValueVariationDetail("overridden", CONTEXT, DEFAULT), "override-value"); + source.setOverrides(Collections.emptyList()); + assertNotReady(client.jsonValueVariationDetail("overridden", CONTEXT, DEFAULT)); + } + } + + @Test + public void overrideTakesPrecedenceOverLaunchDarklyDataAndOtherFlagsAreUnaffected() throws Exception { + Iterable>> ldData = flagsOnly( + valueFlag("shared", "ld-value"), valueFlag("plain", "plain-value")); + TestOverrideSource source = new TestOverrideSource(flagsOnly(valueFlag("shared", "override-value"))); + try (LDClient client = new LDClient(SDK_KEY, initializedConfig(ldData, source))) { + assertTrue(client.isInitialized()); + assertOverrideServed(client.jsonValueVariationDetail("shared", CONTEXT, DEFAULT), "override-value"); + EvaluationDetail plain = client.jsonValueVariationDetail("plain", CONTEXT, DEFAULT); + assertEquals(LDValue.of("plain-value"), plain.getValue()); + assertEquals(EvaluationReason.fallthrough(), plain.getReason()); + assertFalse(plain.getReason().isOverrideAffected()); + + // Removing the override returns the flag to LaunchDarkly data. + source.setOverrides(Collections.emptyList()); + EvaluationDetail restored = client.jsonValueVariationDetail("shared", CONTEXT, DEFAULT); + assertEquals(LDValue.of("ld-value"), restored.getValue()); + assertFalse(restored.getReason().isOverrideAffected()); + } + } + + @Test + public void clientWithoutOverrideSourceBehavesAsBefore() throws Exception { + Iterable>> ldData = flagsOnly(valueFlag("plain", "plain-value")); + LDConfig config = configWith(Components.dataSystem().custom().initializers(initializerWith(ldData))).build(); + try (LDClient client = new LDClient(SDK_KEY, config)) { + assertNull(client.dataSystem.getOverrideLayer()); + assertEquals(LDValue.of("plain-value"), client.jsonValueVariation("plain", CONTEXT, DEFAULT)); + } + } + + @Test + public void allFlagsStateContainsOnlyOverridesWhenClientIsNotInitialized() throws Exception { + TestOverrideSource source = new TestOverrideSource(flagsOnly(valueFlag("overridden", "override-value"))); + try (LDClient client = new LDClient(SDK_KEY, uninitializedConfig(source))) { + FeatureFlagsState state = client.allFlagsState(CONTEXT, FlagsStateOption.WITH_REASONS); + assertTrue(state.isValid()); + assertEquals(Collections.singletonMap("overridden", LDValue.of("override-value")), state.toValuesMap()); + assertTrue(state.getFlagReason("overridden").isOverrideAffected()); + } + } + + @Test + public void clientSideAllFlagsStateKeepsADefinitionWithClientSideButNotAFlagValuesEntry() throws Exception { + // A flagValues entry is not available to client-side SDKs. A full definition that sets + // clientSide is, so that is how an operator overrides a flag that client-side SDKs use. + try (TempDir dir = TempDir.create()) { + Path file = dir.getPath().resolve("overrides.json"); + Files.write(file, ("{\"flagValues\": {\"value-flag\": true}, \"flags\": {\"full-flag\": {" + + "\"key\": \"full-flag\", \"version\": 1, \"on\": true, \"variations\": [true], " + + "\"fallthrough\": {\"variation\": 0}, \"clientSide\": true}}}").getBytes(StandardCharsets.UTF_8)); + try (LDClient client = new LDClient(SDK_KEY, uninitializedConfig(FileOverrides.source().filePaths(file)))) { + assertEquals(new HashSet<>(Arrays.asList("full-flag", "value-flag")), + client.allFlagsState(CONTEXT).toValuesMap().keySet()); + FeatureFlagsState clientSide = client.allFlagsState(CONTEXT, FlagsStateOption.CLIENT_SIDE_ONLY); + assertTrue(clientSide.isValid()); + assertEquals(Collections.singletonMap("full-flag", LDValue.of(true)), clientSide.toValuesMap()); + } + } + } + + @Test + public void allFlagsStateOverridesOnlyWarningIsLoggedOnce() throws Exception { + TestOverrideSource source = new TestOverrideSource(flagsOnly(valueFlag("overridden", "override-value"))); + try (LDClient client = new LDClient(SDK_KEY, uninitializedConfig(source))) { + client.allFlagsState(CONTEXT); + client.allFlagsState(CONTEXT); + String message = "WARN:allFlagsState() was called before client initialized; returning only flags from the override layer." + + " This message is logged once."; + assertEquals(1, logCapture.getMessageStrings().stream().filter(m -> m.equals(message)).count()); + } + } + + @Test + public void allFlagsStateIsInvalidWhenNotInitializedAndOverrideLayerIsEmpty() throws Exception { + TestOverrideSource source = new TestOverrideSource(Collections.emptyList()); + try (LDClient client = new LDClient(SDK_KEY, uninitializedConfig(source))) { + FeatureFlagsState state = client.allFlagsState(CONTEXT); + assertFalse(state.isValid()); + assertThat(logCapture.getMessageStrings(), hasItem( + "WARN:allFlagsState() was called before client initialized; data store unavailable, returning no data")); + } + } + + // A client that reads LaunchDarkly data from the given store and never initializes. + private LDConfig uninitializedConfigWithStore(DataStore store, ComponentConfigurer source) { + return configWith(Components.dataSystem().custom().synchronizers(hangingSynchronizer()) + .persistentStore(context -> store, DataStoreMode.READ_ONLY).overrides(source)) + .startWait(Duration.ZERO) + .build(); + } + + @Test + public void allFlagsStateWithoutOverrideFlagsIgnoresDataInAStoreWithoutAnInitMarker() throws Exception { + // A layer that holds no flags changes nothing: the state is invalid, as it is without a source, + // and does not report the LaunchDarkly data in the store. That holds for an empty layer and for + // one that holds only segments. + FixedDataStore store = new FixedDataStore(flagsOnly(valueFlag("launchdarkly-flag", "ld-value")), false); + TestOverrideSource emptySource = new TestOverrideSource(Collections.emptyList()); + try (LDClient client = new LDClient(SDK_KEY, uninitializedConfigWithStore(store, emptySource))) { + FeatureFlagsState state = client.allFlagsState(CONTEXT); + assertFalse(state.isValid()); + assertEquals(Collections.emptyMap(), state.toValuesMap()); + } + TestOverrideSource segmentSource = new TestOverrideSource(OverrideLayerTest.collections( + Collections.emptyList(), Collections.singletonList(segmentBuilder("segment").version(1).build()))); + try (LDClient client = new LDClient(SDK_KEY, uninitializedConfigWithStore(store, segmentSource))) { + FeatureFlagsState state = client.allFlagsState(CONTEXT); + assertFalse(state.isValid()); + assertEquals(Collections.emptyMap(), state.toValuesMap()); + } + } + + @Test + public void allFlagsStateBeforeInitializationExcludesLaunchDarklyDataFromAStoreWithoutAnInitMarker() throws Exception { + FixedDataStore store = new FixedDataStore(flagsOnly(valueFlag("launchdarkly-flag", "ld-value")), false); + TestOverrideSource source = new TestOverrideSource(flagsOnly(valueFlag("overridden", "override-value"))); + try (LDClient client = new LDClient(SDK_KEY, uninitializedConfigWithStore(store, source))) { + FeatureFlagsState state = client.allFlagsState(CONTEXT); + assertTrue(state.isValid()); + assertEquals(Collections.singletonMap("overridden", LDValue.of("override-value")), state.toValuesMap()); + } + } + + @Test + public void allFlagsStateWhenTheStoreFailsHoldsTheOverridesAndIsInvalid() throws Exception { + // A daemon-mode client whose store holds an init marker but cannot list its flags logs the + // failure. Without a source the state is invalid and empty; the overrides add to it, so it holds + // the overridden flags and stays invalid. + FixedDataStore store = new FixedDataStore(flagsOnly(), true); + store.getAllError = new RuntimeException("store failed"); + TestOverrideSource source = new TestOverrideSource(flagsOnly(valueFlag("overridden", "override-value"))); + try (LDClient client = new LDClient(SDK_KEY, daemonConfig(store, source))) { + FeatureFlagsState state = client.allFlagsState(CONTEXT, FlagsStateOption.WITH_REASONS); + assertFalse(state.isValid()); + assertEquals(Collections.singletonMap("overridden", LDValue.of("override-value")), state.toValuesMap()); + assertTrue(state.getFlagReason("overridden").isOverrideAffected()); + assertTrue(logCapture.getMessageStrings().stream().anyMatch(m -> + m.startsWith("ERROR:Exception from data store when evaluating all flags") && m.contains("store failed"))); + } + } + + @Test + public void allFlagsStateWhenTheStoreFailsAndTheLayerHoldsNoFlagsIsInvalidAndEmpty() throws Exception { + FixedDataStore store = new FixedDataStore(flagsOnly(), true); + store.getAllError = new RuntimeException("store failed"); + try (LDClient client = new LDClient(SDK_KEY, daemonConfig(store, new TestOverrideSource(Collections.emptyList())))) { + FeatureFlagsState state = client.allFlagsState(CONTEXT); + assertFalse(state.isValid()); + assertEquals(Collections.emptyMap(), state.toValuesMap()); + assertTrue(logCapture.getMessageStrings().stream().anyMatch(m -> + m.startsWith("ERROR:Exception from data store when evaluating all flags") && m.contains("store failed"))); + } + } + + private LDConfig daemonConfig(DataStore store, TestOverrideSource source) { + return configWith(Components.dataSystem().daemon(context -> store).overrides(source)) + .startWait(Duration.ZERO) + .build(); + } + + @Test + public void allFlagsStateTurnsOffEventTrackingForOverrideAffectedFlags() throws Exception { + long debugUntil = System.currentTimeMillis() + 100000; + FeatureFlag ldTracked = flagBuilder("tracked-overridden").version(100).on(false).offVariation(0) + .variations(LDValue.of("ld-value")).trackEvents(true).debugEventsUntilDate(debugUntil).build(); + FeatureFlag ldPlain = flagBuilder("plain").version(100).on(false).offVariation(0) + .variations(LDValue.of("plain-value")).trackEvents(true).debugEventsUntilDate(debugUntil).build(); + FeatureFlag overrideOnly = flagBuilder("override-only").version(1).on(true).fallthroughVariation(0).offVariation(0) + .variations(LDValue.of("override-only-value")).trackEvents(true).trackEventsFallthrough(true) + .debugEventsUntilDate(debugUntil).build(); + TestOverrideSource source = new TestOverrideSource(flagsOnly( + valueFlag("tracked-overridden", "override-value"), overrideOnly)); + + try (LDClient client = new LDClient(SDK_KEY, initializedConfig(flagsOnly(ldTracked, ldPlain), source))) { + FeatureFlagsState state = client.allFlagsState(CONTEXT, FlagsStateOption.WITH_REASONS); + LDValue json = LDValue.parse(JsonSerialization.serialize(state)); + LDValue flagsState = json.get("$flagsState"); + + // An override-affected flag keeps its value, version, and marked reason, but every tracking + // field is off. + assertEquals(LDValue.of("override-value"), json.get("tracked-overridden")); + LDValue overridden = flagsState.get("tracked-overridden"); + assertEquals(LDValue.of(1), overridden.get("version")); + assertTrue(overridden.get("reason").get("overrideAffected").booleanValue()); + assertTrue(overridden.get("trackEvents").isNull()); + assertTrue(overridden.get("trackReason").isNull()); + assertTrue(overridden.get("debugEventsUntilDate").isNull()); + + // The same holds for a flag that exists only in the override layer, whatever its definition + // requests, including reason tracking from an experiment-like fallthrough. + LDValue only = flagsState.get("override-only"); + assertTrue(only.get("trackEvents").isNull()); + assertTrue(only.get("trackReason").isNull()); + assertTrue(only.get("debugEventsUntilDate").isNull()); + + // A flag with no override keeps its tracking. + LDValue plain = flagsState.get("plain"); + assertTrue(plain.get("trackEvents").booleanValue()); + assertEquals(LDValue.of(debugUntil), plain.get("debugEventsUntilDate")); + assertFalse(plain.get("reason").get("overrideAffected").booleanValue()); + } + } + + @Test + public void flagTrackerIsNotifiedOfOverrideChanges() throws Exception { + // "dependent" depends on "target" in LaunchDarkly data. Overriding "target" affects both. + FeatureFlag target = valueFlag("target", "ld-value"); + FeatureFlag dependent = flagBuilder("dependent").version(1).on(true).fallthroughVariation(0).offVariation(0) + .variations(LDValue.of("x")).prerequisites(prerequisite("target", 0)).build(); + FeatureFlag unrelated = valueFlag("unrelated", "u"); + TestOverrideSource source = new TestOverrideSource(Collections.emptyList()); + + try (LDClient client = new LDClient(SDK_KEY, initializedConfig(flagsOnly(target, dependent, unrelated), source))) { + BlockingQueue events = new LinkedBlockingQueue<>(); + client.getFlagTracker().addFlagChangeListener(events::add); + + source.setOverrides(flagsOnly(valueFlag("target", "override-value"))); + assertEquals(keys("target", "dependent"), awaitKeys(events, 2)); + assertNull(events.poll(200, TimeUnit.MILLISECONDS)); + + source.setOverrides(flagsOnly(valueFlag("target", "override-value-2"))); + assertEquals(keys("target", "dependent"), awaitKeys(events, 2)); + + source.setOverrides(Collections.emptyList()); + assertEquals(keys("target", "dependent"), awaitKeys(events, 2)); + assertNull(events.poll(200, TimeUnit.MILLISECONDS)); + } + } + + @Test + public void flagValueChangeListenerSeesOverrideChanges() throws Exception { + TestOverrideSource source = new TestOverrideSource(Collections.emptyList()); + try (LDClient client = new LDClient(SDK_KEY, initializedConfig(flagsOnly(valueFlag("flag", "ld-value")), source))) { + BlockingQueue values = new LinkedBlockingQueue<>(); + client.getFlagTracker().addFlagValueChangeListener("flag", CONTEXT, e -> values.add(e.getNewValue())); + + source.setOverrides(flagsOnly(valueFlag("flag", "override-value"))); + assertEquals(LDValue.of("override-value"), values.poll(5, TimeUnit.SECONDS)); + + source.setOverrides(Collections.emptyList()); + assertEquals(LDValue.of("ld-value"), values.poll(5, TimeUnit.SECONDS)); + } + } + + @Test + public void segmentOverrideNotifiesFlagsThatReferenceIt() throws Exception { + FeatureFlag usesSegment = flagBuilder("uses-segment").version(1).on(true).fallthroughVariation(0).offVariation(0) + .variations(LDValue.of("no"), LDValue.of("yes")) + .rules(ruleBuilder().id("r").variation(1).clauses(clauseMatchingSegment("seg")).build()).build(); + Segment ldSegment = segmentBuilder("seg").version(1).build(); + TestOverrideSource source = new TestOverrideSource(Collections.emptyList()); + + try (LDClient client = new LDClient(SDK_KEY, initializedConfig( + collections(Collections.singletonList(usesSegment), Collections.singletonList(ldSegment)), source))) { + assertEquals(LDValue.of("no"), client.jsonValueVariation("uses-segment", CONTEXT, DEFAULT)); + BlockingQueue events = new LinkedBlockingQueue<>(); + client.getFlagTracker().addFlagChangeListener(events::add); + + Segment overrideSegment = segmentBuilder("seg").version(2).included(CONTEXT.getKey()).build(); + source.setOverrides(collections(Collections.emptyList(), Collections.singletonList(overrideSegment))); + + assertEquals(keys("uses-segment"), awaitKeys(events, 1)); + EvaluationDetail detail = client.jsonValueVariationDetail("uses-segment", CONTEXT, DEFAULT); + assertEquals(LDValue.of("yes"), detail.getValue()); + assertTrue(detail.getReason().isOverrideAffected()); + } + } + + @Test + public void overrideSourceIsStartedBeforeConstructorReturnsAndClosedWithClient() throws Exception { + TestOverrideSource source = new TestOverrideSource(flagsOnly(valueFlag("flag", "override-value"))); + LDClient client = new LDClient(SDK_KEY, initializedConfig(flagsOnly(valueFlag("flag", "ld-value")), source)); + try { + assertTrue(source.started.get()); + assertFalse(source.closed.get()); + assertEquals(LDValue.of("override-value"), client.jsonValueVariation("flag", CONTEXT, DEFAULT)); + } finally { + client.close(); + } + assertTrue(source.closed.get()); + } + + @Test + public void overrideSourceStartFailureIsLoggedAndTheClientRunsWithoutOverrides() throws Exception { + // The SDK's own source never throws from start. A custom source could, and that must not keep + // the client from starting: the failure is logged, the source is closed, and the client runs + // without overrides. + TestOverrideSource source = new TestOverrideSource(flagsOnly(valueFlag("flag", "override-value"))); + source.startError = new RuntimeException("source failed"); + try (LDClient client = new LDClient(SDK_KEY, uninitializedConfig(source))) { + assertTrue(source.closed.get()); + assertTrue(logCapture.getMessageStrings().stream().anyMatch(m -> + m.startsWith("ERROR:Unable to start the override source; the client runs without overrides") + && m.contains("source failed"))); + assertNotReady(client.jsonValueVariationDetail("flag", CONTEXT, DEFAULT)); + } + } + + @Test + public void overrideSourceThatSuppliesASnapshotAndThenFailsToStartLeavesNoOverrides() throws Exception { + // The snapshot the source supplied before it failed is removed with the source, so the client + // runs without overrides, as the log says, and its all-flags state agrees. + TestOverrideSource source = new TestOverrideSource(flagsOnly(valueFlag("flag", "override-value"))); + source.startError = new RuntimeException("source failed"); + source.supplyDataBeforeStartError = true; + try (LDClient client = new LDClient(SDK_KEY, uninitializedConfig(source))) { + assertTrue(source.closed.get()); + assertNotReady(client.jsonValueVariationDetail("flag", CONTEXT, DEFAULT)); + assertFalse(client.allFlagsState(CONTEXT).isValid()); + } + } + + @Test + public void overrideSourceCloseFailureDoesNotStopTheClientFromShuttingDown() throws Exception { + FixedDataStore store = new FixedDataStore(flagsOnly(), false); + TestOverrideSource source = new TestOverrideSource(Collections.emptyList()); + source.closeError = new RuntimeException("close failed"); + LDConfig config = configWith(Components.dataSystem().custom().synchronizers(hangingSynchronizer()) + .persistentStore(context -> store, DataStoreMode.READ_WRITE).overrides(source)) + .startWait(Duration.ZERO) + .build(); + LDClient client = new LDClient(SDK_KEY, config); + + client.close(); + + assertTrue(source.closed.get()); + assertTrue(store.closed); + assertTrue(logCapture.getMessageStrings().stream().anyMatch(m -> + m.startsWith("WARN:Unable to close the override source") && m.contains("close failed"))); + } + + @Test + public void snapshotSuppliedAfterTheClientIsClosedIsIgnored() throws Exception { + TestOverrideSource source = new TestOverrideSource(Collections.emptyList()); + LDClient client = new LDClient(SDK_KEY, uninitializedConfig(source)); + client.close(); + + source.deliverEvenIfClosed(flagsOnly(valueFlag("late-flag", "override-value"))); + + assertNull(client.dataSystem.getOverrideLayer().get(DataModel.FEATURES, "late-flag")); + } + + @Test + public void overrideSourceIsBuiltWithClientContext() throws Exception { + TestOverrideSource source = new TestOverrideSource(Collections.emptyList()); + try (LDClient client = new LDClient(SDK_KEY, initializedConfig(flagsOnly(), source))) { + assertEquals(SDK_KEY, source.buildContext.getSdkKey()); + } + } + + @Test + public void offlineClientDoesNotStartOverrideSource() throws Exception { + TestOverrideSource source = new TestOverrideSource(flagsOnly(valueFlag("flag", "override-value"))); + LDConfig config = configWith(Components.dataSystem().custom().overrides(source)).offline(true).build(); + try (LDClient client = new LDClient(SDK_KEY, config)) { + assertFalse(source.started.get()); + assertNull(client.dataSystem.getOverrideLayer()); + assertEquals(DEFAULT, client.jsonValueVariation("flag", CONTEXT, DEFAULT)); + assertThat(logCapture.getMessageStrings(), hasItem( + "WARN:Offline mode is enabled; the override source is not started and no overrides are served")); + } + assertFalse(source.closed.get()); + } + + @Test + public void invalidOverrideSourceConfigurationFailsClientConstruction() { + ComponentConfigurer broken = context -> { + throw new IllegalArgumentException("no file paths were specified"); + }; + LDConfig config = configWith(Components.dataSystem().custom().initializers(initializerWith(flagsOnly())).overrides(broken)) + .build(); + try { + new LDClient(SDK_KEY, config).close(); + fail("expected exception"); + } catch (IllegalArgumentException e) { + assertEquals("no file paths were specified", e.getMessage()); + } catch (Exception e) { + fail("unexpected exception " + e); + } + } + + @Test + public void isFlagKnownSeesOverrideOnlyFlagWhenInitialized() throws Exception { + TestOverrideSource source = new TestOverrideSource(flagsOnly(valueFlag("override-only", "v"))); + try (LDClient client = new LDClient(SDK_KEY, initializedConfig(flagsOnly(valueFlag("plain", "p")), source))) { + assertTrue(client.isFlagKnown("override-only")); + assertTrue(client.isFlagKnown("plain")); + assertFalse(client.isFlagKnown("missing")); + } + } + + @Test + public void overrideLayerDoesNotAffectInitializationStatus() throws Exception { + TestOverrideSource source = new TestOverrideSource(flagsOnly(valueFlag("flag", "override-value"))); + try (LDClient client = new LDClient(SDK_KEY, uninitializedConfig(source))) { + assertFalse(client.isInitialized()); + assertThat(client.getDataSourceStatusProvider().getStatus().getState().toString(), not("VALID")); + } + } + + private static Set keys(String... keys) { + Set s = new HashSet<>(); + Collections.addAll(s, keys); + return s; + } + + private static Set awaitKeys(BlockingQueue events, int count) throws InterruptedException { + Set keys = new HashSet<>(); + for (int i = 0; i < count; i++) { + FlagChangeEvent e = events.poll(5, TimeUnit.SECONDS); + if (e == null) { + throw new AssertionError("expected " + count + " flag change events but got " + keys); + } + keys.add(e.getKey()); + } + return keys; + } + + @Test + public void isFlagKnownReportsAnOverrideBeforeTheClientIsInitialized() throws Exception { + TestOverrideSource source = new TestOverrideSource(flagsOnly(valueFlag("overridden", "v"))); + try (LDClient client = new LDClient(SDK_KEY, uninitializedConfig(source))) { + assertFalse(client.isInitialized()); + assertTrue(client.isFlagKnown("overridden")); + assertFalse(client.isFlagKnown("other")); + } + } + + @Test + public void launchDarklyChangeNotifiesOverridesThatDependOnIt() throws Exception { + // The override "overridden" has the LaunchDarkly flag "prereq" as a prerequisite. A change to + // prereq on the LaunchDarkly side notifies the override too. + TestDataV2 td = TestDataV2.synchronizer(); + td.update(td.flag("prereq").on(true)); + FeatureFlag overridden = flagBuilder("overridden").version(1).on(true).fallthroughVariation(0).offVariation(0) + .variations(LDValue.of("x")).prerequisites(prerequisite("prereq", 0)).build(); + TestOverrideSource source = new TestOverrideSource(flagsOnly(overridden)); + LDConfig config = configWith(Components.dataSystem().custom().synchronizers(td).overrides(source)).build(); + try (LDClient client = new LDClient(SDK_KEY, config)) { + BlockingQueue events = new LinkedBlockingQueue<>(); + client.getFlagTracker().addFlagChangeListener(events::add); + td.update(td.flag("prereq").on(false)); + assertEquals(keys("prereq", "overridden"), awaitKeys(events, 2)); + assertNull(events.poll(200, TimeUnit.MILLISECONDS)); + } + } + + // A store with fixed contents. Constructed not initialized, it is like a persistent store that holds + // LaunchDarkly data but no init marker. getAll throws getAllError when it is set. + private static final class FixedDataStore implements DataStore { + private final Iterable>> data; + private final boolean initialized; + volatile RuntimeException getAllError; + volatile boolean closed; + + FixedDataStore(Iterable>> data, boolean initialized) { + this.data = data; + this.initialized = initialized; + } + + @Override + public void init(FullDataSet allData) { + } + + @Override + public ItemDescriptor get(DataKind kind, String key) { + for (Map.Entry e : items(kind)) { + if (e.getKey().equals(key)) { + return e.getValue(); + } + } + return null; + } + + @Override + public KeyedItems getAll(DataKind kind) { + if (getAllError != null) { + throw getAllError; + } + return new KeyedItems<>(items(kind)); + } + + @Override + public boolean upsert(DataKind kind, String key, ItemDescriptor item) { + return true; + } + + @Override + public boolean isInitialized() { + return initialized; + } + + @Override + public boolean isStatusMonitoringEnabled() { + return false; + } + + @Override + public CacheStats getCacheStats() { + return null; + } + + @Override + public void close() { + closed = true; + } + + private Iterable> items(DataKind kind) { + for (Map.Entry> e : data) { + if (e.getKey() == kind) { + return e.getValue().getItems(); + } + } + return Collections.emptyList(); + } + } +} diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/OverrideLayerTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/OverrideLayerTest.java new file mode 100644 index 00000000..de4f46cb --- /dev/null +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/OverrideLayerTest.java @@ -0,0 +1,181 @@ +package com.launchdarkly.sdk.server; + +import com.launchdarkly.sdk.server.DataModel.FeatureFlag; +import com.launchdarkly.sdk.server.DataModelDependencies.KindAndKey; +import com.launchdarkly.sdk.server.DataModel.Segment; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.DataKind; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.KeyedItems; + +import org.junit.Test; + +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.HashSet; + +import static com.launchdarkly.sdk.server.DataModel.FEATURES; +import static com.launchdarkly.sdk.server.DataModel.SEGMENTS; +import static com.launchdarkly.sdk.server.ModelBuilders.flagBuilder; +import static com.launchdarkly.sdk.server.ModelBuilders.ruleBuilder; +import static com.launchdarkly.sdk.server.ModelBuilders.prerequisite; +import static com.launchdarkly.sdk.server.ModelBuilders.clauseMatchingSegment; +import static com.launchdarkly.sdk.server.ModelBuilders.segmentBuilder; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +@SuppressWarnings("javadoc") +public class OverrideLayerTest { + static Iterable>> collections( + Iterable flags, Iterable segments) { + List> flagItems = new ArrayList<>(); + for (FeatureFlag f : flags) { + flagItems.add(new AbstractMap.SimpleEntry<>(f.getKey(), new ItemDescriptor(f.getVersion(), f))); + } + List> segmentItems = new ArrayList<>(); + for (Segment s : segments) { + segmentItems.add(new AbstractMap.SimpleEntry<>(s.getKey(), new ItemDescriptor(s.getVersion(), s))); + } + List>> result = new ArrayList<>(); + result.add(new AbstractMap.SimpleEntry<>(FEATURES, new KeyedItems<>(flagItems))); + result.add(new AbstractMap.SimpleEntry<>(SEGMENTS, new KeyedItems<>(segmentItems))); + return result; + } + + static Iterable>> flagsOnly(FeatureFlag... flags) { + return collections(java.util.Arrays.asList(flags), Collections.emptyList()); + } + + @Test + public void newLayerIsEmpty() { + OverrideLayer layer = new OverrideLayer(); + assertTrue(layer.isEmpty()); + assertNull(layer.get(FEATURES, "flag")); + assertTrue(layer.all(FEATURES).isEmpty()); + } + + @Test + public void setAllStoresMarkedCopiesWithoutMutatingSource() { + OverrideLayer layer = new OverrideLayer(); + FeatureFlag flag = flagBuilder("flag1").version(2).build(); + Segment segment = segmentBuilder("segment1").version(3).build(); + + layer.setAll(collections(Collections.singletonList(flag), Collections.singletonList(segment))); + + assertFalse(flag.isOverride()); + assertFalse(segment.isOverride()); + assertFalse(layer.isEmpty()); + + ItemDescriptor storedFlag = layer.get(FEATURES, "flag1"); + assertNotNull(storedFlag); + assertEquals(2, storedFlag.getVersion()); + assertTrue(((FeatureFlag) storedFlag.getItem()).isOverride()); + assertNotSame(flag, storedFlag.getItem()); + assertSame(flag.preprocessed, ((FeatureFlag) storedFlag.getItem()).preprocessed); + + ItemDescriptor storedSegment = layer.get(SEGMENTS, "segment1"); + assertNotNull(storedSegment); + assertEquals(3, storedSegment.getVersion()); + assertTrue(((Segment) storedSegment.getItem()).isOverride()); + } + + @Test + public void setAllReplacesEverything() { + OverrideLayer layer = new OverrideLayer(); + layer.setAll(flagsOnly(flagBuilder("a").build(), flagBuilder("b").build())); + assertNotNull(layer.get(FEATURES, "a")); + assertNotNull(layer.get(FEATURES, "b")); + + OverrideLayer.Replacement replacement = layer.setAll(flagsOnly(flagBuilder("b").build(), flagBuilder("c").build())); + assertNull(layer.get(FEATURES, "a")); + assertNotNull(layer.get(FEATURES, "b")); + assertNotNull(layer.get(FEATURES, "c")); + assertEquals(2, replacement.previous.get(FEATURES).size()); + assertEquals(2, replacement.current.get(FEATURES).size()); + assertTrue(replacement.previous.get(FEATURES).containsKey("a")); + assertTrue(replacement.current.get(FEATURES).containsKey("c")); + } + + @Test + public void emptyOrNullSnapshotClearsLayer() { + OverrideLayer layer = new OverrideLayer(); + layer.setAll(flagsOnly(flagBuilder("a").build())); + assertFalse(layer.isEmpty()); + + layer.setAll(Collections.emptyList()); + assertTrue(layer.isEmpty()); + assertNull(layer.get(FEATURES, "a")); + + layer.setAll(flagsOnly(flagBuilder("a").build())); + layer.setAll(null); + assertTrue(layer.isEmpty()); + } + + @Test + public void deletedPlaceholderIsStoredAsIs() { + OverrideLayer layer = new OverrideLayer(); + ItemDescriptor deleted = ItemDescriptor.deletedItem(5); + layer.setAll(Collections.singletonList(new AbstractMap.SimpleEntry<>(FEATURES, + new KeyedItems<>(Collections.singletonList(new AbstractMap.SimpleEntry<>("gone", deleted)))))); + assertSame(deleted, layer.get(FEATURES, "gone")); + } + + @Test + public void allReturnsEntriesOfKindOnly() { + OverrideLayer layer = new OverrideLayer(); + layer.setAll(collections(Collections.singletonList(flagBuilder("f").build()), + Collections.singletonList(segmentBuilder("s").build()))); + assertEquals(1, layer.all(FEATURES).size()); + assertEquals(1, layer.all(SEGMENTS).size()); + assertTrue(layer.all(FEATURES).containsKey("f")); + assertTrue(layer.all(SEGMENTS).containsKey("s")); + } + + private static Set affectedFrom(OverrideLayer layer, DataKind kind, String key) { + Set affected = new HashSet<>(); + layer.addAffectedItems(affected, new KindAndKey(kind, key)); + return affected; + } + + private static Set items(KindAndKey... items) { + return new HashSet<>(java.util.Arrays.asList(items)); + } + + @Test + public void reportsEntriesThatDependOnAnItem() { + // a matches a segment; b has a as a prerequisite, so b depends on the segment through a. + FeatureFlag a = flagBuilder("a").version(1).on(true) + .rules(ruleBuilder().id("r").variation(0).clauses(clauseMatchingSegment("seg")).build()).build(); + FeatureFlag b = flagBuilder("b").version(1).on(true).prerequisites(prerequisite("a", 0)).build(); + OverrideLayer layer = new OverrideLayer(); + layer.setAll(flagsOnly(a, b)); + + assertEquals(items(new KindAndKey(SEGMENTS, "seg"), new KindAndKey(FEATURES, "a"), new KindAndKey(FEATURES, "b")), + affectedFrom(layer, SEGMENTS, "seg")); + assertEquals(items(new KindAndKey(FEATURES, "a"), new KindAndKey(FEATURES, "b")), + affectedFrom(layer, FEATURES, "a")); + assertEquals(items(new KindAndKey(SEGMENTS, "unreferenced")), affectedFrom(layer, SEGMENTS, "unreferenced")); + } + + @Test + public void forgetsEdgesOfReplacedEntries() { + FeatureFlag a = flagBuilder("a").version(1).on(true) + .rules(ruleBuilder().id("r").variation(0).clauses(clauseMatchingSegment("seg")).build()).build(); + FeatureFlag c = flagBuilder("c").version(1).on(true) + .rules(ruleBuilder().id("r").variation(0).clauses(clauseMatchingSegment("seg2")).build()).build(); + OverrideLayer layer = new OverrideLayer(); + layer.setAll(flagsOnly(a)); + layer.setAll(flagsOnly(c)); + + assertEquals(items(new KindAndKey(SEGMENTS, "seg")), affectedFrom(layer, SEGMENTS, "seg")); + assertEquals(items(new KindAndKey(SEGMENTS, "seg2"), new KindAndKey(FEATURES, "c")), affectedFrom(layer, SEGMENTS, "seg2")); + } +} diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/OverrideOverlayStoreTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/OverrideOverlayStoreTest.java new file mode 100644 index 00000000..2edb0982 --- /dev/null +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/OverrideOverlayStoreTest.java @@ -0,0 +1,187 @@ +package com.launchdarkly.sdk.server; + +import com.launchdarkly.sdk.server.DataModel.FeatureFlag; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.DataKind; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.KeyedItems; + +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static com.launchdarkly.sdk.server.DataModel.FEATURES; +import static com.launchdarkly.sdk.server.DataModel.SEGMENTS; +import static com.launchdarkly.sdk.server.DataStoreTestTypes.toItemsMap; +import static com.launchdarkly.sdk.server.ModelBuilders.flagBuilder; +import static com.launchdarkly.sdk.server.OverrideLayerTest.flagsOnly; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@SuppressWarnings("javadoc") +public class OverrideOverlayStoreTest { + /** + * A base store with arbitrary data, initialization state, and an optional read failure. + */ + static final class FakeBaseStore implements ReadOnlyStore { + final Map> data = new HashMap<>(); + boolean initialized = true; + RuntimeException getAllError = null; + int getAllCalls = 0; + + FakeBaseStore withFlag(FeatureFlag flag) { + data.computeIfAbsent(FEATURES, k -> new HashMap<>()).put(flag.getKey(), new ItemDescriptor(flag.getVersion(), flag)); + return this; + } + + FakeBaseStore withDeletedFlag(String key, int version) { + data.computeIfAbsent(FEATURES, k -> new HashMap<>()).put(key, ItemDescriptor.deletedItem(version)); + return this; + } + + @Override + public ItemDescriptor get(DataKind kind, String key) { + Map items = data.get(kind); + return items == null ? null : items.get(key); + } + + @Override + public KeyedItems getAll(DataKind kind) { + getAllCalls++; + if (getAllError != null) { + throw getAllError; + } + Map items = data.get(kind); + return new KeyedItems<>(items == null ? null : items.entrySet()); + } + + @Override + public boolean isInitialized() { + return initialized; + } + } + + private static FeatureFlag flagOf(ItemDescriptor item) { + return (FeatureFlag) item.getItem(); + } + + @Test + public void getPrefersOverrideEntry() { + FeatureFlag ldFlag = flagBuilder("shared").version(1).on(true).build(); + FeatureFlag ldOnly = flagBuilder("ld-only").version(1).build(); + FeatureFlag overrideFlag = flagBuilder("shared").version(2).on(false).build(); + FeatureFlag overrideOnly = flagBuilder("override-only").version(1).build(); + FakeBaseStore base = new FakeBaseStore().withFlag(ldFlag).withFlag(ldOnly); + OverrideLayer layer = new OverrideLayer(); + layer.setAll(flagsOnly(overrideFlag, overrideOnly)); + OverrideOverlayStore overlay = new OverrideOverlayStore(base, layer); + + ItemDescriptor shared = overlay.get(FEATURES, "shared"); + assertEquals(2, shared.getVersion()); + assertTrue(flagOf(shared).isOverride()); + assertFalse(flagOf(shared).isOn()); + + ItemDescriptor onlyLd = overlay.get(FEATURES, "ld-only"); + assertFalse(flagOf(onlyLd).isOverride()); + assertSame(ldOnly, onlyLd.getItem()); + + assertTrue(flagOf(overlay.get(FEATURES, "override-only")).isOverride()); + assertNull(overlay.get(FEATURES, "missing")); + assertNull(overlay.get(SEGMENTS, "shared")); + } + + @Test + public void getServesOverridesFromUninitializedBase() { + FakeBaseStore base = new FakeBaseStore(); + base.initialized = false; + OverrideLayer layer = new OverrideLayer(); + layer.setAll(flagsOnly(flagBuilder("flag").build())); + OverrideOverlayStore overlay = new OverrideOverlayStore(base, layer); + + assertFalse(overlay.isInitialized()); + assertTrue(flagOf(overlay.get(FEATURES, "flag")).isOverride()); + assertNull(overlay.get(FEATURES, "other")); + } + + @Test + public void isInitializedDelegatesToBase() { + FakeBaseStore base = new FakeBaseStore(); + OverrideLayer layer = new OverrideLayer(); + OverrideOverlayStore overlay = new OverrideOverlayStore(base, layer); + assertTrue(overlay.isInitialized()); + base.initialized = false; + assertFalse(overlay.isInitialized()); + layer.setAll(flagsOnly(flagBuilder("flag").build())); + assertFalse(overlay.isInitialized()); + } + + @Test + public void getAllIsUnionWithOverridePrecedence() { + FakeBaseStore base = new FakeBaseStore() + .withFlag(flagBuilder("shared").version(1).build()) + .withFlag(flagBuilder("ld-only").version(1).build()) + .withDeletedFlag("tombstoned", 9); + OverrideLayer layer = new OverrideLayer(); + layer.setAll(flagsOnly( + flagBuilder("shared").version(2).build(), + flagBuilder("override-only").version(1).build(), + flagBuilder("tombstoned").version(1).build())); + OverrideOverlayStore overlay = new OverrideOverlayStore(base, layer); + + Map all = toItemsMap(overlay.getAll(FEATURES)); + assertEquals(4, all.size()); + assertEquals(2, all.get("shared").getVersion()); + assertTrue(flagOf(all.get("shared")).isOverride()); + assertFalse(flagOf(all.get("ld-only")).isOverride()); + assertTrue(flagOf(all.get("override-only")).isOverride()); + // The override wins over a deleted-item placeholder in the base. + assertTrue(flagOf(all.get("tombstoned")).isOverride()); + } + + @Test + public void getAllWithEmptyLayerReturnsBaseResultAsIs() { + FakeBaseStore base = new FakeBaseStore().withFlag(flagBuilder("a").build()); + OverrideOverlayStore overlay = new OverrideOverlayStore(base, new OverrideLayer()); + KeyedItems all = overlay.getAll(FEATURES); + assertEquals(1, toItemsMap(all).size()); + assertFalse(flagOf(toItemsMap(all).get("a")).isOverride()); + } + + @Test + public void getAllPropagatesBaseFailureWhenLayerHoldsEntries() { + // A result of the layer's entries alone would look complete to a caller such as the all-flags + // state, so the failure propagates as it does without the overlay. A per-key read still serves + // the override. + FakeBaseStore base = new FakeBaseStore(); + RuntimeException error = new RuntimeException("store down"); + base.getAllError = error; + OverrideLayer layer = new OverrideLayer(); + layer.setAll(flagsOnly(flagBuilder("flag").build())); + OverrideOverlayStore overlay = new OverrideOverlayStore(base, layer); + try { + overlay.getAll(FEATURES); + fail("expected exception"); + } catch (RuntimeException e) { + assertSame(error, e); + } + assertTrue(flagOf(overlay.get(FEATURES, "flag")).isOverride()); + } + + @Test + public void getAllPropagatesBaseFailureWhenLayerIsEmpty() { + FakeBaseStore base = new FakeBaseStore(); + RuntimeException error = new RuntimeException("store down"); + base.getAllError = error; + OverrideOverlayStore overlay = new OverrideOverlayStore(base, new OverrideLayer()); + try { + overlay.getAll(FEATURES); + fail("expected exception"); + } catch (RuntimeException e) { + assertSame(error, e); + } + } +} diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/OverrideSinkImplTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/OverrideSinkImplTest.java new file mode 100644 index 00000000..ca7a3916 --- /dev/null +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/OverrideSinkImplTest.java @@ -0,0 +1,229 @@ +package com.launchdarkly.sdk.server; + +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.server.DataModel.FeatureFlag; +import com.launchdarkly.sdk.server.DataModel.Segment; +import com.launchdarkly.sdk.server.OverrideOverlayStoreTest.FakeBaseStore; +import com.launchdarkly.sdk.server.interfaces.FlagChangeEvent; +import com.launchdarkly.sdk.server.interfaces.FlagChangeListener; + +import org.junit.Test; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import static com.launchdarkly.sdk.server.ModelBuilders.clauseMatchingSegment; +import static com.launchdarkly.sdk.server.ModelBuilders.flagBuilder; +import static com.launchdarkly.sdk.server.ModelBuilders.prerequisite; +import static com.launchdarkly.sdk.server.ModelBuilders.ruleBuilder; +import static com.launchdarkly.sdk.server.ModelBuilders.segmentBuilder; +import static com.launchdarkly.sdk.server.OverrideLayerTest.collections; +import static com.launchdarkly.sdk.server.OverrideLayerTest.flagsOnly; +import static com.launchdarkly.sdk.server.TestComponents.nullLogger; +import static com.launchdarkly.sdk.server.TestComponents.sharedExecutor; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +@SuppressWarnings("javadoc") +public class OverrideSinkImplTest { + private final FakeBaseStore base = new FakeBaseStore(); + private final OverrideLayer layer = new OverrideLayer(); + private final EventBroadcasterImpl broadcaster = + EventBroadcasterImpl.forFlagChangeEvents(sharedExecutor, nullLogger); + private final OverrideSinkImpl sink = new OverrideSinkImpl(layer, base, broadcaster, nullLogger); + private final BlockingQueue events = new LinkedBlockingQueue<>(); + + private void listen() { + broadcaster.register(events::add); + } + + private Set awaitKeys(int count) throws InterruptedException { + Set keys = new HashSet<>(); + for (int i = 0; i < count; i++) { + FlagChangeEvent e = events.poll(5, TimeUnit.SECONDS); + if (e == null) { + throw new AssertionError("expected " + count + " flag change events but got " + keys); + } + keys.add(e.getKey()); + } + return keys; + } + + private void assertNoMoreEvents() throws InterruptedException { + assertNull(events.poll(200, TimeUnit.MILLISECONDS)); + } + + private static Set keys(String... keys) { + Set s = new HashSet<>(); + Collections.addAll(s, keys); + return s; + } + + @Test + public void setOverridesReplacesLayer() { + sink.setOverrides(flagsOnly(flagBuilder("a").build())); + assertTrue(((FeatureFlag) layer.get(DataModel.FEATURES, "a").getItem()).isOverride()); + sink.setOverrides(Collections.emptyList()); + assertTrue(layer.isEmpty()); + } + + @Test + public void closeKeepsTheLayerAndIgnoresLaterSnapshots() throws Exception { + listen(); + sink.setOverrides(flagsOnly(flagBuilder("a").version(1).build())); + assertEquals(keys("a"), awaitKeys(1)); + + sink.close(); + assertNotNull(layer.get(DataModel.FEATURES, "a")); + assertNoMoreEvents(); + + // A snapshot after closing changes nothing. + sink.setOverrides(flagsOnly(flagBuilder("b").version(1).build())); + assertNotNull(layer.get(DataModel.FEATURES, "a")); + assertNull(layer.get(DataModel.FEATURES, "b")); + assertNoMoreEvents(); + } + + @Test + public void closeAndClearClearsTheLayerAndIgnoresLaterSnapshots() throws Exception { + listen(); + sink.setOverrides(flagsOnly(flagBuilder("a").version(1).build())); + assertEquals(keys("a"), awaitKeys(1)); + + // Closing removes what the source supplied, and the removal is a change. + sink.closeAndClear(); + assertTrue(layer.isEmpty()); + assertEquals(keys("a"), awaitKeys(1)); + + // A snapshot after closing changes nothing. + sink.setOverrides(flagsOnly(flagBuilder("a").version(1).build())); + assertTrue(layer.isEmpty()); + assertNoMoreEvents(); + } + + @Test + public void notifiesOnAddChangeAndRemove() throws Exception { + listen(); + + sink.setOverrides(flagsOnly(flagBuilder("a").version(1).on(true).build())); + assertEquals(keys("a"), awaitKeys(1)); + + // The same content again is not a change. + sink.setOverrides(flagsOnly(flagBuilder("a").version(1).on(true).build())); + assertNoMoreEvents(); + + // A content change at the same version is a change. + sink.setOverrides(flagsOnly(flagBuilder("a").version(1).on(false).build())); + assertEquals(keys("a"), awaitKeys(1)); + + // A version change is a change. + sink.setOverrides(flagsOnly(flagBuilder("a").version(2).on(false).build())); + assertEquals(keys("a"), awaitKeys(1)); + + // Removal is a change. + sink.setOverrides(Collections.emptyList()); + assertEquals(keys("a"), awaitKeys(1)); + assertNoMoreEvents(); + } + + @Test + public void addedOverrideIdenticalToLaunchDarklyDataIsStillAChange() throws Exception { + FeatureFlag ldFlag = flagBuilder("a").version(1).on(true).build(); + base.withFlag(ldFlag); + listen(); + + sink.setOverrides(flagsOnly(flagBuilder("a").version(1).on(true).build())); + assertEquals(keys("a"), awaitKeys(1)); + } + + @Test + public void segmentOverrideFansOutToDependentFlags() throws Exception { + FeatureFlag usesSegment = flagBuilder("uses-segment").on(true) + .rules(ruleBuilder().id("r").variation(0).clauses(clauseMatchingSegment("seg")).build()).build(); + FeatureFlag unrelated = flagBuilder("unrelated").build(); + base.withFlag(usesSegment).withFlag(unrelated); + listen(); + + Segment segment = segmentBuilder("seg").included("u").build(); + sink.setOverrides(collections(Collections.emptyList(), Collections.singletonList(segment))); + + assertEquals(keys("uses-segment"), awaitKeys(1)); + assertNoMoreEvents(); + } + + @Test + public void prerequisiteFanOutUsesOldAndNewViews() throws Exception { + // The LaunchDarkly definition of "parent" depends on "prereq". The override of "parent" removes + // that dependency. Adding the override affects "parent". Later changing the override of + // "prereq" does not affect "parent" while the parent override is in place. Removing the parent + // override restores the edge, and "parent" is affected again. + FeatureFlag ldParent = flagBuilder("parent").on(true).prerequisites(prerequisite("prereq", 0)).build(); + FeatureFlag ldPrereq = flagBuilder("prereq").on(true).build(); + base.withFlag(ldParent).withFlag(ldPrereq); + listen(); + + FeatureFlag parentOverride = flagBuilder("parent").version(5).on(true).build(); + sink.setOverrides(flagsOnly(parentOverride)); + assertEquals(keys("parent"), awaitKeys(1)); + + sink.setOverrides(flagsOnly(parentOverride, flagBuilder("prereq").version(7).on(false).build())); + assertEquals(keys("prereq"), awaitKeys(1)); + assertNoMoreEvents(); + + // Removing both overrides: "prereq" changes, and "parent" depends on it in the new view. + sink.setOverrides(Collections.emptyList()); + assertEquals(keys("parent", "prereq"), awaitKeys(2)); + assertNoMoreEvents(); + } + + @Test + public void removingOverrideThatReferencedSegmentAffectsOnlyThatFlag() throws Exception { + // The override of "flag" depends on segment "seg" that exists only in the base. Removing the + // override affects "flag" through the old view. Nothing else is affected. + base.withFlag(flagBuilder("flag").build()); + listen(); + FeatureFlag override = flagBuilder("flag").on(true) + .rules(ruleBuilder().id("r").variation(0).clauses(clauseMatchingSegment("seg")).build()).build(); + sink.setOverrides(flagsOnly(override)); + assertEquals(keys("flag"), awaitKeys(1)); + + sink.setOverrides(Collections.emptyList()); + assertEquals(keys("flag"), awaitKeys(1)); + assertNoMoreEvents(); + } + + @Test + public void skipsDiffWorkWithoutListeners() { + base.withFlag(flagBuilder("a").build()); + sink.setOverrides(flagsOnly(flagBuilder("a").version(2).build())); + assertEquals(0, base.getAllCalls); + assertEquals(2, layer.get(DataModel.FEATURES, "a").getVersion()); + } + + @Test + public void toleratesBaseReadFailure() throws Exception { + base.getAllError = new RuntimeException("store down"); + listen(); + + sink.setOverrides(flagsOnly(flagBuilder("a").version(1).build())); + + // The directly changed key is still reported and the layer is updated. + assertEquals(keys("a"), awaitKeys(1)); + assertEquals(1, layer.get(DataModel.FEATURES, "a").getVersion()); + } + + @Test + public void valueChangeInSameVersionIsDetectedThroughSerializedForm() throws Exception { + listen(); + sink.setOverrides(flagsOnly(flagBuilder("a").version(1).variations(LDValue.of("x")).build())); + awaitKeys(1); + sink.setOverrides(flagsOnly(flagBuilder("a").version(1).variations(LDValue.of("y")).build())); + assertEquals(keys("a"), awaitKeys(1)); + } +} diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/OverrideSpecVectorsTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/OverrideSpecVectorsTest.java new file mode 100644 index 00000000..57d7f887 --- /dev/null +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/OverrideSpecVectorsTest.java @@ -0,0 +1,175 @@ +package com.launchdarkly.sdk.server; + +import com.launchdarkly.logging.LDLogLevel; +import com.launchdarkly.sdk.EvaluationDetail; +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.internal.events.Event; +import com.launchdarkly.sdk.json.JsonSerialization; +import com.launchdarkly.sdk.server.TestComponents.TestEventProcessor; +import com.launchdarkly.sdk.server.integrations.DataSystemBuilder; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.DataKind; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.KeyedItems; + +import org.junit.Test; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Scanner; + +import static com.launchdarkly.sdk.server.DataModel.FEATURES; +import static com.launchdarkly.sdk.server.DataModel.SEGMENTS; +import static com.launchdarkly.sdk.server.OverrideTestDataSources.hangingSynchronizer; +import static com.launchdarkly.sdk.server.OverrideTestDataSources.initializerWith; +import static com.launchdarkly.sdk.server.TestComponents.specificComponent; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Runs the OVERRIDE specification's test vectors. Each vector sets up LaunchDarkly data, an + * override layer, and an initialization state, evaluates one flag through the full client stack, + * and checks the value, the variation index, the reason, and the marking that the client hands to the + * event processor for the evaluation. + */ +@SuppressWarnings("javadoc") +public class OverrideSpecVectorsTest extends BaseTest { + private static final String VECTORS_RESOURCE = "/override-vectors/vectors.json"; + // The vectors' semantics are versioned. A schema change means this runner needs review. + private static final String SUPPORTED_SCHEMA_VERSION = "0.4.0"; + + static LDValue loadVectors() { + try (InputStream in = OverrideSpecVectorsTest.class.getResourceAsStream(VECTORS_RESOURCE)) { + if (in == null) { + throw new AssertionError("vectors resource not found: " + VECTORS_RESOURCE); + } + Scanner scanner = new Scanner(in, StandardCharsets.UTF_8.name()).useDelimiter("\\A"); + return LDValue.parse(scanner.hasNext() ? scanner.next() : ""); + } catch (java.io.IOException e) { + throw new AssertionError(e); + } + } + + static Iterable>> collectionsFromJson( + LDValue flags, LDValue flagValues, LDValue segments) { + List> flagItems = new ArrayList<>(); + for (String key : flags.keys()) { + flagItems.add(new AbstractMap.SimpleEntry<>(key, FEATURES.deserialize(flags.get(key).toJsonString()))); + } + for (String key : flagValues.keys()) { + // A value-only entry is a flag that is on and serves the value as its single variation by + // fallthrough. + LDValue flag = LDValue.buildObject().put("key", key).put("version", 0).put("on", true) + .put("variations", LDValue.buildArray().add(flagValues.get(key)).build()) + .put("fallthrough", LDValue.buildObject().put("variation", 0).build()).build(); + flagItems.add(new AbstractMap.SimpleEntry<>(key, FEATURES.deserialize(flag.toJsonString()))); + } + List> segmentItems = new ArrayList<>(); + for (String key : segments.keys()) { + segmentItems.add(new AbstractMap.SimpleEntry<>(key, SEGMENTS.deserialize(segments.get(key).toJsonString()))); + } + List>> result = new ArrayList<>(); + result.add(new AbstractMap.SimpleEntry<>(FEATURES, new KeyedItems<>(flagItems))); + result.add(new AbstractMap.SimpleEntry<>(SEGMENTS, new KeyedItems<>(segmentItems))); + return result; + } + + @Test + public void vectorsFileHasSupportedSchema() { + LDValue file = loadVectors(); + assertEquals(SUPPORTED_SCHEMA_VERSION, file.get("schemaVersion").stringValue()); + assertTrue(file.get("vectors").size() > 0); + } + + @Test + public void allVectorsPass() throws Exception { + LDValue file = loadVectors(); + List failures = new ArrayList<>(); + for (LDValue vector : file.get("vectors").values()) { + String name = vector.get("group").stringValue() + ": " + vector.get("description").stringValue(); + try { + runVector(vector); + } catch (AssertionError e) { + failures.add(name + " -> " + e.getMessage()); + } + } + assertTrue("failing vectors:\n" + String.join("\n", failures), failures.isEmpty()); + } + + private void runVector(LDValue vector) throws Exception { + LDValue ldData = vector.get("launchDarklyData"); + boolean initialized = ldData.get("initialized").booleanValue(); + LDValue overrides = vector.get("overrides"); + TestOverrideSource source = new TestOverrideSource(collectionsFromJson( + overrides.get("flags"), overrides.get("flagValues"), overrides.get("segments"))); + + DataSystemBuilder dataSystem = Components.dataSystem().custom().overrides(source); + TestEventProcessor events = new TestEventProcessor(); + LDConfig.Builder config = new LDConfig.Builder() + .events(specificComponent(events)) + .logging(Components.logging(testLogging).level(LDLogLevel.DEBUG)); + if (initialized) { + dataSystem.initializers(initializerWith(collectionsFromJson( + ldData.get("flags"), LDValue.buildObject().build(), ldData.get("segments")))); + } else { + // With no sources at all, the client would consider itself initialized with empty data rather + // than apply its not-initialized handling. A synchronizer that never delivers anything avoids + // that. + dataSystem.synchronizers(hangingSynchronizer()); + config.startWait(Duration.ZERO); + } + config.dataSystem(dataSystem); + + LDValue evaluate = vector.get("evaluate"); + LDContext context = JsonSerialization.deserialize(evaluate.get("context").toJsonString(), LDContext.class); + LDValue expect = vector.get("expect"); + + try (LDClient client = new LDClient("sdk-key", config.build())) { + assertEquals("initialization state", initialized, client.isInitialized()); + EvaluationDetail detail = client.jsonValueVariationDetail( + evaluate.get("flagKey").stringValue(), context, evaluate.get("defaultValue")); + + assertEquals("value", expect.get("value"), detail.getValue()); + if (expect.get("variationIndex").isNull()) { + assertTrue("variationIndex should be undefined", detail.isDefaultValue()); + } else { + assertEquals("variationIndex", expect.get("variationIndex").intValue(), detail.getVariationIndex()); + } + assertVectorReason(expect.get("reason"), LDValue.parse(JsonSerialization.serialize(detail.getReason()))); + + // summaryOverrideAffected is the marking the client hands to the event processor for this + // evaluation. The event processor keys individual event suppression and the summary counter + // marker on that value, not on the reason. + if (!expect.get("summaryOverrideAffected").isNull()) { + List records = new ArrayList<>(); + for (Event e : events.events) { + if (e instanceof Event.FeatureRequest + && ((Event.FeatureRequest) e).getKey().equals(evaluate.get("flagKey").stringValue())) { + records.add((Event.FeatureRequest) e); + } + } + assertEquals("expected exactly one evaluation record for the flag", 1, records.size()); + assertEquals("summaryOverrideAffected", expect.get("summaryOverrideAffected").booleanValue(), + records.get(0).isOverrideAffected()); + } + } + } + + // Compares the actual reason against only the fields present in the expected reason. The + // override indicator collapses tri-state: an expected reason that omits it requires the actual + // reason to report false, which is never serialized, or to omit it. + private static void assertVectorReason(LDValue expected, LDValue actual) { + for (String field : expected.keys()) { + assertEquals("reason field " + field, expected.get(field), actual.get(field)); + } + if (expected.get("overrideAffected").isNull()) { + assertFalse("overrideAffected must be false or omitted", actual.get("overrideAffected").booleanValue()); + } + } +} diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/OverrideTestDataSources.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/OverrideTestDataSources.java new file mode 100644 index 00000000..2499bc3a --- /dev/null +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/OverrideTestDataSources.java @@ -0,0 +1,76 @@ +package com.launchdarkly.sdk.server; + +import com.launchdarkly.sdk.fdv2.ChangeSet; +import com.launchdarkly.sdk.fdv2.ChangeSetType; +import com.launchdarkly.sdk.fdv2.Selector; +import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; +import com.launchdarkly.sdk.server.datasources.Initializer; +import com.launchdarkly.sdk.server.datasources.Synchronizer; +import com.launchdarkly.sdk.server.subsystems.DataSourceBuilder; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.DataKind; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.KeyedItems; + +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +/** + * FDv2 data sources for tests of the override layer: an initializer that supplies fixed + * LaunchDarkly data as a full transfer, and a synchronizer that never supplies anything so the + * client never initializes. + */ +@SuppressWarnings("javadoc") +abstract class OverrideTestDataSources { + private OverrideTestDataSources() {} + + /** + * Returns an initializer that supplies the given data as a full transfer with a selector, which + * is what makes the client report that it is initialized. + */ + static DataSourceBuilder initializerWith( + final Iterable>> data) { + return context -> new Initializer() { + @Override + public String name() { + return "TestInitializer"; + } + + @Override + public CompletableFuture run() { + ChangeSet>>> changeSet = new ChangeSet<>( + ChangeSetType.Full, Selector.make(1, "test-state"), data, null, false); + return CompletableFuture.completedFuture(FDv2SourceResult.changeSet(changeSet, false)); + } + + @Override + public void close() { + } + }; + } + + /** + * Returns a synchronizer that never delivers anything. With no sources at all the client would + * consider itself initialized with empty data. A synchronizer that never delivers keeps the + * client in its not-initialized state. + */ + static DataSourceBuilder hangingSynchronizer() { + return context -> new Synchronizer() { + private final CompletableFuture shutdown = new CompletableFuture<>(); + + @Override + public String name() { + return "HangingSynchronizer"; + } + + @Override + public CompletableFuture next() { + return shutdown; + } + + @Override + public void close() { + shutdown.complete(FDv2SourceResult.shutdown()); + } + }; + } +} diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/TestComponents.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/TestComponents.java index 77b4d699..5bc8d51a 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/TestComponents.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/TestComponents.java @@ -149,8 +149,17 @@ public void recordEvaluationEvent(LDContext context, String flagKey, int flagVer EvaluationReason reason, LDValue defaultValue, String prerequisiteOfFlagKey, boolean requireFullEvent, Long debugEventsUntilDate, boolean excludeFromSummaries, Long samplingRatio) { + recordEvaluationEvent(context, flagKey, flagVersion, variation, value, reason, defaultValue, prerequisiteOfFlagKey, + requireFullEvent, debugEventsUntilDate, excludeFromSummaries, samplingRatio, false); + } + + @Override + public void recordEvaluationEvent(LDContext context, String flagKey, int flagVersion, int variation, LDValue value, + EvaluationReason reason, LDValue defaultValue, String prerequisiteOfFlagKey, boolean requireFullEvent, + Long debugEventsUntilDate, boolean excludeFromSummaries, Long samplingRatio, boolean overrideAffected) { events.add(new Event.FeatureRequest(System.currentTimeMillis(), flagKey, context, flagVersion, - variation, value, defaultValue, reason, prerequisiteOfFlagKey, requireFullEvent, debugEventsUntilDate, false)); + variation, value, defaultValue, reason, prerequisiteOfFlagKey, requireFullEvent, debugEventsUntilDate, false, + samplingRatio == null ? 1 : samplingRatio, excludeFromSummaries, overrideAffected)); } @Override diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/TestOverrideSource.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/TestOverrideSource.java new file mode 100644 index 00000000..3d4e1853 --- /dev/null +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/TestOverrideSource.java @@ -0,0 +1,86 @@ +package com.launchdarkly.sdk.server; + +import com.launchdarkly.sdk.server.subsystems.ClientContext; +import com.launchdarkly.sdk.server.subsystems.ComponentConfigurer; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.DataKind; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.KeyedItems; +import com.launchdarkly.sdk.server.subsystems.OverrideSink; +import com.launchdarkly.sdk.server.subsystems.OverrideSource; + +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * A test override source. It supplies its initial data when started and forwards later + * replacements to the sink, as if the source's backing data had changed. + */ +@SuppressWarnings("javadoc") +public final class TestOverrideSource implements OverrideSource, ComponentConfigurer { + private final Iterable>> initialData; + private volatile OverrideSink sink; + final AtomicBoolean started = new AtomicBoolean(false); + final AtomicBoolean closed = new AtomicBoolean(false); + volatile ClientContext buildContext; + // When set, start throws it instead of supplying data. + volatile RuntimeException startError; + // When set with startError, start supplies the data to the sink and then throws. + volatile boolean supplyDataBeforeStartError; + // When set, close throws it after recording the close. + volatile RuntimeException closeError; + + public TestOverrideSource(Iterable>> initialData) { + this.initialData = initialData; + } + + @Override + public OverrideSource build(ClientContext clientContext) { + this.buildContext = clientContext; + return this; + } + + @Override + public void start(OverrideSink sink) { + if (startError != null) { + if (supplyDataBeforeStartError && initialData != null) { + sink.setOverrides(initialData); + } + throw startError; + } + this.sink = sink; + started.set(true); + if (initialData != null) { + sink.setOverrides(initialData); + } + } + + /** + * Replaces the override layer contents. Must be called after the source was started. + * + * @param data the complete set of override entries + */ + public void setOverrides(Iterable>> data) { + OverrideSink s = sink; + if (s != null && !closed.get()) { + s.setOverrides(data); + } + } + + /** + * Supplies a snapshot to the sink even after the source was closed, as a misbehaving source or a + * load that races the close could. + * + * @param data the complete set of override entries + */ + public void deliverEvenIfClosed(Iterable>> data) { + sink.setOverrides(data); + } + + @Override + public void close() { + closed.set(true); + if (closeError != null) { + throw closeError; + } + } +} diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/FileDataReloaderEmptiedFileTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/FileDataReloaderEmptiedFileTest.java new file mode 100644 index 00000000..561d8b40 --- /dev/null +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/FileDataReloaderEmptiedFileTest.java @@ -0,0 +1,680 @@ +package com.launchdarkly.sdk.server.integrations; + +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.logging.LogCapture; +import com.launchdarkly.logging.Logs; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.server.integrations.FileDataReloader.EmptiedFileException; +import com.launchdarkly.sdk.server.integrations.FileDataSourceParsing.FileDataException; +import com.launchdarkly.sdk.server.integrations.OverrideFileLoader.LoadResult; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.DataKind; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.KeyedItems; +import com.launchdarkly.testhelpers.TempDir; + +import org.junit.After; +import org.junit.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongSupplier; + +import static com.launchdarkly.sdk.server.DataModel.FEATURES; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.not; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * The reloader with the override files loader, on real files that are rewritten in place: a file + * whose content is in effect and is found empty is deferred until it has stayed empty for the retry + * delay. + */ +@SuppressWarnings("javadoc") +public class FileDataReloaderEmptiedFileTest { + // Long enough that a test finishes what it does while a file is empty before the retry reads the + // file again, also on a loaded machine. + private static final Duration LONG_RETRY = Duration.ofMillis(1500); + private static final long WAIT_MILLIS = 10000; + private static final String REREAD_LOG = "DEBUG:Reading flag data again after a file was found empty"; + + private final LogCapture logCapture = Logs.capture(); + private final LDLogger logger = LDLogger.withAdapter(logCapture, ""); + private final List reloaders = new ArrayList<>(); + + @After + public void closeReloaders() { + for (FileDataReloader r : reloaders) { + r.close(); + } + } + + /** + * Records the value of the flag "flag" in each applied snapshot, or "none" when a snapshot does + * not have it, and every reported failure. + */ + private static final class Recorder implements FileDataReloader.Handler { + final List applied = Collections.synchronizedList(new ArrayList<>()); + final List errors = Collections.synchronizedList(new ArrayList<>()); + + @Override + public void apply(LoadResult result) { + String value = "none"; + for (Map.Entry> kind : result.getData()) { + if (kind.getKey() != FEATURES) { + continue; + } + for (Map.Entry item : kind.getValue().getItems()) { + if (item.getKey().equals("flag")) { + LDValue v = LDValue.parse(FEATURES.serialize(item.getValue())).get("variations").get(0); + value = v.isString() ? v.stringValue() : v.toJsonString(); + } + } + } + applied.add(value); + } + + @Override + public void onError(FileDataException e) { + errors.add(e); + } + } + + /** + * The override files loader, counting the loads that have finished. + */ + private static final class CountingLoader implements FileDataReloader.Loader { + final OverrideFileLoader loader; + final AtomicInteger finished = new AtomicInteger(); + + CountingLoader(Path... paths) { + loader = new OverrideFileLoader(Arrays.asList(paths), FileData.DuplicateKeysHandling.FAIL); + } + + CountingLoader(LongSupplier clock, Path... paths) { + loader = new OverrideFileLoader(Arrays.asList(paths), FileData.DuplicateKeysHandling.FAIL, clock); + } + + @Override + public LoadResult load(Duration retryDelay) throws FileDataException, EmptiedFileException { + try { + return loader.load(retryDelay); + } finally { + finished.incrementAndGet(); + } + } + + @Override + public void accepted(LoadResult result) { + loader.accepted(result); + } + + void awaitFinished(int expected) throws InterruptedException { + long deadline = System.currentTimeMillis() + WAIT_MILLIS; + while (finished.get() < expected) { + if (System.currentTimeMillis() > deadline) { + throw new AssertionError("timed out waiting for load " + expected + ", got " + finished.get()); + } + Thread.sleep(5); + } + } + } + + private FileDataReloader reloader(CountingLoader loader, Recorder recorder, Duration retryDelay) { + FileDataReloader r = new FileDataReloader(loader, recorder, logger, Duration.ZERO, retryDelay, true); + reloaders.add(r); + return r; + } + + private static void awaitSize(List list, int expected) throws InterruptedException { + long deadline = System.currentTimeMillis() + WAIT_MILLIS; + while (list.size() < expected) { + if (System.currentTimeMillis() > deadline) { + throw new AssertionError("timed out waiting for size " + expected + ", got " + list); + } + Thread.sleep(5); + } + } + + private static Path write(TempDir dir, String name, String contents) throws Exception { + Path p = dir.getPath().resolve(name); + Files.write(p, contents.getBytes()); + return p; + } + + private static String yaml(String value) { + return "flagValues:\n flag: \"" + value + "\"\n"; + } + + private static String json(String value) { + return "{\"flagValues\":{\"flag\":\"" + value + "\"}}"; + } + + private static String readAgainLog(Path file) { + return "DEBUG:File " + file + " is empty, possibly while it is being rewritten; reading it again before applying it"; + } + + // For the tests that move a clock themselves: a retry delay so long that no retry fires during the + // test, so every load is one the test starts with reloadNow. + private static final Duration HOUR = Duration.ofHours(1); + + private static final long TENTH = HOUR.toNanos() / 10; + + // Writes the file again, in place, with a later modification time than the previous write. File + // system timestamps can be coarse, so it waits until the time differs. + private static void writeAgain(Path file, String contents) throws Exception { + long before = Files.getLastModifiedTime(file).toMillis(); + long deadline = System.currentTimeMillis() + 5000; + do { + Thread.sleep(20); + InPlaceWrites.rewrite(file, "x", () -> { }); + InPlaceWrites.rewrite(file, contents, () -> { }); + } while (Files.getLastModifiedTime(file).toMillis() == before && System.currentTimeMillis() < deadline); + } + + @Test + public void holdAsksForTheRetryWhenTheDelayIsOver() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", yaml("a")); + AtomicLong clock = new AtomicLong(); + FileDataReloader r = reloader(new CountingLoader(clock::get, file), new Recorder(), HOUR); + r.reloadNow(); + + InPlaceWrites.truncate(file); + r.reloadNow(); + assertEquals(HOUR.toNanos() + TENTH, r.getLastRequestedRetryNanos()); + + // A later load asks for the rest of the delay, not a whole one. + clock.addAndGet(Duration.ofMinutes(30).toNanos()); + r.reloadNow(); + assertEquals(Duration.ofMinutes(30).toNanos() + TENTH, r.getLastRequestedRetryNanos()); + } + } + + @Test + public void holdAsksForTheRetryWhenTheBoundIsOver() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", yaml("a")); + AtomicLong clock = new AtomicLong(); + Recorder recorder = new Recorder(); + FileDataReloader r = reloader(new CountingLoader(clock::get, file), recorder, HOUR); + r.reloadNow(); + + InPlaceWrites.truncate(file); + r.reloadNow(); + for (int i = 0; i < 4; i++) { + clock.addAndGet(Duration.ofMinutes(60).toNanos()); + writeAgain(file, i % 2 == 0 ? " " : " "); + r.reloadNow(); + } + clock.addAndGet(Duration.ofMinutes(48).toNanos()); + writeAgain(file, " "); + r.reloadNow(); + + // The file's delay would be over in an hour, but its bound is over in 12 minutes. + assertEquals(Collections.singletonList("a"), recorder.applied); + assertEquals(Duration.ofMinutes(12).toNanos() + TENTH, r.getLastRequestedRetryNanos()); + } + } + + @Test + public void holdOfSeveralFilesAsksForTheRetryWhenTheFirstIsDue() throws Exception { + try (TempDir dir = TempDir.create()) { + Path a = write(dir, "a.yaml", yaml("a")); + Path b = write(dir, "b.yaml", "flagValues:\n other: \"b\"\n"); + AtomicLong clock = new AtomicLong(); + FileDataReloader r = reloader(new CountingLoader(clock::get, a, b), new Recorder(), HOUR); + r.reloadNow(); + + InPlaceWrites.truncate(a); + r.reloadNow(); + clock.addAndGet(Duration.ofMinutes(20).toNanos()); + InPlaceWrites.truncate(b); + r.reloadNow(); + assertEquals(Duration.ofMinutes(40).toNanos() + TENTH, r.getLastRequestedRetryNanos()); + } + } + + @Test + public void holdStartedByAChangeSignalAsksForTheRetryWhenTheDelayIsOver() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", yaml("a")); + AtomicLong clock = new AtomicLong(); + CountingLoader loader = new CountingLoader(clock::get, file); + FileDataReloader r = reloader(loader, new Recorder(), HOUR); + r.reloadNow(); + + InPlaceWrites.truncate(file); + r.trigger(); + loader.awaitFinished(2); + long deadline = System.currentTimeMillis() + WAIT_MILLIS; + while (r.getLastRequestedRetryNanos() != HOUR.toNanos() + TENTH && System.currentTimeMillis() < deadline) { + Thread.sleep(5); + } + assertEquals(HOUR.toNanos() + TENTH, r.getLastRequestedRetryNanos()); + assertThat(logCapture.getMessageStrings(), hasItem(readAgainLog(file))); + } + } + + @Test + public void failedLoadAsksForTheRetryDelay() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", yaml("a")); + AtomicLong clock = new AtomicLong(); + Recorder recorder = new Recorder(); + FileDataReloader r = reloader(new CountingLoader(clock::get, file), recorder, HOUR); + r.reloadNow(); + + InPlaceWrites.rewrite(file, "flagValues: [", () -> { }); + r.reloadNow(); + assertEquals(1, recorder.errors.size()); + assertEquals(HOUR.toNanos(), r.getLastRequestedRetryNanos()); + } + } + + @Test + public void holdEndsExactlyAfterTheRetryDelay() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", yaml("a")); + AtomicLong clock = new AtomicLong(); + CountingLoader loader = new CountingLoader(clock::get, file); + Recorder recorder = new Recorder(); + FileDataReloader r = reloader(loader, recorder, HOUR); + r.reloadNow(); + + InPlaceWrites.truncate(file); + r.reloadNow(); + clock.addAndGet(HOUR.toNanos() * 95 / 100); + r.reloadNow(); + assertEquals(Collections.singletonList("a"), recorder.applied); + clock.addAndGet(HOUR.toNanos() * 5 / 100); + r.reloadNow(); + assertEquals(Arrays.asList("a", "none"), recorder.applied); + assertTrue(recorder.errors.isEmpty()); + } + } + + @Test + public void fileRewrittenWithTheSameContentIsHeldBackAgainLater() throws Exception { + // A tool rewrites the file in place now and then with the same content. The load after a rewrite + // is skipped as unchanged, and that must start the file's bound over as an application does, or + // a rewrite five retry delays later would be applied while the file is empty. + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", yaml("a")); + AtomicLong clock = new AtomicLong(); + CountingLoader loader = new CountingLoader(clock::get, file); + Recorder recorder = new Recorder(); + FileDataReloader r = reloader(loader, recorder, HOUR); + r.reloadNow(); + + InPlaceWrites.truncate(file); + r.reloadNow(); + InPlaceWrites.rewrite(file, yaml("a"), () -> { }); + r.reloadNow(); + assertEquals(3, loader.finished.get()); + assertEquals(Collections.singletonList("a"), recorder.applied); + + clock.addAndGet(HOUR.toNanos() * 6); + InPlaceWrites.truncate(file); + r.reloadNow(); + assertEquals(Collections.singletonList("a"), recorder.applied); + assertTrue(recorder.errors.isEmpty()); + } + } + + @Test + public void sameBytesMovedToAFileOfAnotherFormatAreApplied() throws Exception { + // The format depends on the path, so the load after the move is not skipped as unchanged: YAML + // reads on as true, and JSON reads it as a string. + try (TempDir dir = TempDir.create()) { + Path yaml = write(dir, "overrides.yaml", "{\"flagValues\": {\"flag\": on}}"); + Path json = dir.getPath().resolve("overrides.json"); + Recorder recorder = new Recorder(); + FileDataReloader r = reloader(new CountingLoader(yaml, json), recorder, HOUR); + r.reloadNow(); + Files.move(yaml, json); + r.reloadNow(); + assertEquals(Arrays.asList("true", "on"), recorder.applied); + } + } + + @Test + public void changesToOtherFilesWaitWhileAFileIsHeld() throws Exception { + // A reload applies all the files or none. + try (TempDir dir = TempDir.create()) { + Path emptied = write(dir, "emptied.yaml", "flagValues:\n other: \"x\"\n"); + Path changed = write(dir, "changed.yaml", yaml("b1")); + AtomicLong clock = new AtomicLong(); + CountingLoader loader = new CountingLoader(clock::get, emptied, changed); + Recorder recorder = new Recorder(); + FileDataReloader r = reloader(loader, recorder, HOUR); + r.reloadNow(); + + InPlaceWrites.truncate(emptied); + InPlaceWrites.rewrite(changed, yaml("b2"), () -> { }); + r.reloadNow(); + assertEquals(Collections.singletonList("b1"), recorder.applied); + clock.addAndGet(HOUR.toNanos()); + r.reloadNow(); + assertEquals(Arrays.asList("b1", "b2"), recorder.applied); + } + } + + @Test + public void emptiedYamlFileIsReadAgainAndTheRewrittenContentIsApplied() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", yaml("a")); + CountingLoader loader = new CountingLoader(file); + Recorder recorder = new Recorder(); + FileDataReloader r = reloader(loader, recorder, LONG_RETRY); + r.reloadNow(); + + InPlaceWrites.rewrite(file, yaml("b"), () -> { + // The change signal of the truncation starts a load that finds the file empty. + r.trigger(); + loader.awaitFinished(2); + }); + r.trigger(); + awaitSize(recorder.applied, 2); + + assertEquals(Arrays.asList("a", "b"), recorder.applied); + assertTrue(recorder.errors.isEmpty()); + assertThat(logCapture.getMessageStrings(), hasItem(readAgainLog(file))); + } + } + + @Test + public void yamlFileThatStaysEmptyIsAppliedByTheRetry() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", yaml("a")); + CountingLoader loader = new CountingLoader(file); + Recorder recorder = new Recorder(); + FileDataReloader r = reloader(loader, recorder, LONG_RETRY); + r.reloadNow(); + + InPlaceWrites.truncate(file); + r.trigger(); + loader.awaitFinished(2); + assertEquals(Collections.singletonList("a"), recorder.applied); + + // No further signal arrives. The retry reads the file again and applies it. + awaitSize(recorder.applied, 2); + assertEquals(Arrays.asList("a", "none"), recorder.applied); + assertEquals(3, loader.finished.get()); + assertTrue(recorder.errors.isEmpty()); + assertThat(logCapture.getMessageStrings(), hasItem(REREAD_LOG)); + } + } + + @Test + public void emptiedJsonFileIsReportedOnlyByTheLoadAfterTheOneThatFindsItEmpty() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.json", json("a")); + CountingLoader loader = new CountingLoader(file); + Recorder recorder = new Recorder(); + FileDataReloader r = reloader(loader, recorder, LONG_RETRY); + r.reloadNow(); + + InPlaceWrites.truncate(file); + r.trigger(); + loader.awaitFinished(2); + assertTrue(recorder.errors.isEmpty()); + + awaitSize(recorder.errors, 1); + assertThat(FileDataReloader.describe(recorder.errors.get(0)), + containsString("the file has no content, which is not a JSON document")); + assertEquals(Collections.singletonList("a"), recorder.applied); + } + } + + @Test + public void furtherChangeSignalWithinTheGraceDoesNotApplyTheEmptyFile() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", yaml("a")); + CountingLoader loader = new CountingLoader(file); + Recorder recorder = new Recorder(); + FileDataReloader r = reloader(loader, recorder, LONG_RETRY); + r.reloadNow(); + + InPlaceWrites.truncate(file); + r.trigger(); + loader.awaitFinished(2); + r.trigger(); + loader.awaitFinished(3); + assertEquals(Collections.singletonList("a"), recorder.applied); + + // The retry armed by the second load finds that the file has stayed empty for the retry delay. + awaitSize(recorder.applied, 2); + assertEquals(Arrays.asList("a", "none"), recorder.applied); + assertEquals(4, loader.finished.get()); + assertThat(logCapture.getMessageStrings(), hasItem(REREAD_LOG)); + } + } + + @Test + public void secondRewriteWithinOneDebounceWindowIsDeferredToo() throws Exception { + // F1: the write of b and the next truncation happen before any load reads b. The next load + // finds the file empty again; the file is in a new rewrite, not one that stayed empty. + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", yaml("a")); + CountingLoader loader = new CountingLoader(file); + Recorder recorder = new Recorder(); + FileDataReloader r = reloader(loader, recorder, LONG_RETRY); + r.reloadNow(); + + InPlaceWrites.rewrite(file, yaml("b"), () -> { + r.trigger(); + loader.awaitFinished(2); + }); + Thread.sleep(50); + InPlaceWrites.rewrite(file, yaml("c"), () -> { + r.trigger(); + loader.awaitFinished(3); + }); + r.trigger(); + awaitSize(recorder.applied, 2); + + assertEquals(Arrays.asList("a", "c"), recorder.applied); + } + } + + @Test + public void changeToAnotherFileWhileAFileIsEmptyDoesNotApplyTheEmptyFile() throws Exception { + // F2: a change to b starts a load while a is still in the middle of its rewrite. + try (TempDir dir = TempDir.create()) { + Path a = write(dir, "a.yaml", yaml("a")); + Path b = write(dir, "b.yaml", "flagValues:\n other: \"b1\"\n"); + CountingLoader loader = new CountingLoader(a, b); + Recorder recorder = new Recorder(); + FileDataReloader r = reloader(loader, recorder, LONG_RETRY); + r.reloadNow(); + + InPlaceWrites.rewrite(a, yaml("a2"), () -> { + r.trigger(); + loader.awaitFinished(2); + InPlaceWrites.rewrite(b, "flagValues:\n other: \"b2\"\n", () -> { }); + r.trigger(); + loader.awaitFinished(3); + }); + r.trigger(); + awaitSize(recorder.applied, 2); + + assertEquals(Arrays.asList("a", "a2"), recorder.applied); + } + } + + @Test + public void retryThatFindsANewTruncationIsDeferredToo() throws Exception { + // F3: the file is rewritten, and rewritten again before any load reads the first new content. + // The retry armed by the first deferral finds the second truncation. + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", yaml("a")); + CountingLoader loader = new CountingLoader(file); + Recorder recorder = new Recorder(); + FileDataReloader r = reloader(loader, recorder, LONG_RETRY); + r.reloadNow(); + + InPlaceWrites.rewrite(file, yaml("b"), () -> { + r.trigger(); + loader.awaitFinished(2); + }); + Thread.sleep(50); + InPlaceWrites.rewrite(file, yaml("c"), () -> loader.awaitFinished(3)); + r.trigger(); + awaitSize(recorder.applied, 2); + + assertEquals(Arrays.asList("a", "c"), recorder.applied); + assertThat(logCapture.getMessageStrings(), hasItem(REREAD_LOG)); + } + } + + @Test + public void filesEmptiedTogetherAreAppliedByTheFirstRetry() throws Exception { + try (TempDir dir = TempDir.create()) { + Path a = write(dir, "a.yaml", yaml("a")); + Path b = write(dir, "b.yaml", "flagValues:\n other: \"b\"\n"); + CountingLoader loader = new CountingLoader(a, b); + Recorder recorder = new Recorder(); + FileDataReloader r = reloader(loader, recorder, LONG_RETRY); + r.reloadNow(); + + InPlaceWrites.truncate(a); + InPlaceWrites.truncate(b); + r.trigger(); + loader.awaitFinished(2); + awaitSize(recorder.applied, 2); + + assertEquals(Arrays.asList("a", "none"), recorder.applied); + assertEquals(3, loader.finished.get()); + assertThat(logCapture.getMessageStrings(), hasItem(readAgainLog(a))); + assertThat(logCapture.getMessageStrings(), hasItem(readAgainLog(b))); + } + } + + @Test + public void fileMovedToAnotherConfiguredPathIsDeferredWhenItIsRewritten() throws Exception { + // Moving a file from one configured path to another leaves the same flags in effect. The load + // after the move records that the content now comes from the new path, or an in-place rewrite + // of that path would be applied while it is empty. The skip-unchanged digest covers the paths, + // so that load is applied rather than skipped. + try (TempDir dir = TempDir.create()) { + Path a = write(dir, "a.yaml", yaml("a")); + Path b = dir.getPath().resolve("b.yaml"); + CountingLoader loader = new CountingLoader(a, b); + Recorder recorder = new Recorder(); + FileDataReloader r = reloader(loader, recorder, LONG_RETRY); + r.reloadNow(); + + Files.move(a, b); + r.trigger(); + awaitSize(recorder.applied, 2); + assertEquals(Arrays.asList("a", "a"), recorder.applied); + + InPlaceWrites.rewrite(b, yaml("b"), () -> { + r.trigger(); + loader.awaitFinished(3); + }); + r.trigger(); + awaitSize(recorder.applied, 3); + + assertEquals(Arrays.asList("a", "a", "b"), recorder.applied); + assertThat(logCapture.getMessageStrings(), hasItem(readAgainLog(b))); + } + } + + @Test + public void emptyPlaceholderFileDoesNotDelayAChangeToAnotherFile() throws Exception { + try (TempDir dir = TempDir.create()) { + Path placeholder = write(dir, "placeholder.yaml", ""); + Path data = write(dir, "data.json", json("a")); + CountingLoader loader = new CountingLoader(placeholder, data); + Recorder recorder = new Recorder(); + FileDataReloader r = reloader(loader, recorder, LONG_RETRY); + r.reloadNow(); + + InPlaceWrites.rewrite(data, json("b"), () -> { }); + r.trigger(); + awaitSize(recorder.applied, 2); + + // The load of the signal applied the change, without waiting for a retry. + assertEquals(Arrays.asList("a", "b"), recorder.applied); + assertEquals(2, loader.finished.get()); + assertThat(logCapture.getMessageStrings(), not(hasItem(readAgainLog(placeholder)))); + } + } + + @Test + public void fileThatIsDeletedAndCreatedEmptyIsAppliedRightAway() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", yaml("a")); + CountingLoader loader = new CountingLoader(file); + Recorder recorder = new Recorder(); + FileDataReloader r = reloader(loader, recorder, LONG_RETRY); + r.reloadNow(); + + Files.delete(file); + r.trigger(); + loader.awaitFinished(2); + Files.write(file, new byte[0]); + r.trigger(); + awaitSize(recorder.applied, 3); + + assertEquals(Arrays.asList("a", "none", "none"), recorder.applied); + assertEquals(3, loader.finished.get()); + assertThat(logCapture.getMessageStrings(), not(hasItem(readAgainLog(file)))); + } + } + + @Test + public void retryLeftOverFromAFailureThatFindsTheFileEmptiedReadsItAgain() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", yaml("a")); + CountingLoader loader = new CountingLoader(file); + Recorder recorder = new Recorder(); + FileDataReloader r = reloader(loader, recorder, LONG_RETRY); + r.reloadNow(); + + // A load fails and arms the retry. + InPlaceWrites.rewrite(file, "flagValues: [", () -> { }); + r.trigger(); + awaitSize(recorder.errors, 1); + + // The file is rewritten again, and only the retry reads it while it is empty. + InPlaceWrites.rewrite(file, yaml("b"), () -> loader.awaitFinished(3)); + r.trigger(); + awaitSize(recorder.applied, 2); + + assertEquals(Arrays.asList("a", "b"), recorder.applied); + assertThat(logCapture.getMessageStrings(), hasItem(readAgainLog(file))); + } + } + + @Test + public void withoutRetriesAnEmptiedFileIsAppliedRightAway() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", yaml("a")); + CountingLoader loader = new CountingLoader(file); + Recorder recorder = new Recorder(); + FileDataReloader r = reloader(loader, recorder, Duration.ZERO); + r.reloadNow(); + + InPlaceWrites.truncate(file); + r.trigger(); + awaitSize(recorder.applied, 2); + + assertEquals(Arrays.asList("a", "none"), recorder.applied); + assertEquals(2, loader.finished.get()); + assertThat(logCapture.getMessageStrings(), not(hasItem(readAgainLog(file)))); + } + } +} diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/FileDataReloaderTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/FileDataReloaderTest.java index 017906bc..8ded2803 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/FileDataReloaderTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/FileDataReloaderTest.java @@ -3,14 +3,18 @@ import com.launchdarkly.logging.LDLogger; import com.launchdarkly.logging.LogCapture; import com.launchdarkly.logging.Logs; +import com.launchdarkly.sdk.server.integrations.FileDataReloader.EmptiedFileException; import com.launchdarkly.sdk.server.integrations.OverrideFileLoader.LoadResult; import com.launchdarkly.sdk.server.integrations.FileDataSourceParsing.FileDataException; import org.junit.After; import org.junit.Test; +import java.nio.file.Path; +import java.nio.file.Paths; import java.time.Duration; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.CountDownLatch; @@ -18,7 +22,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Supplier; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; @@ -26,6 +29,7 @@ import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.startsWith; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -63,14 +67,24 @@ private static FileDataException failure(String message) { return new FileDataException(message, null, null); } + /** + * One step of a scripted loader: it returns a result or throws. + */ + private interface Step { + LoadResult run() throws FileDataException, EmptiedFileException; + } + /** * A scripted loader: each call takes the next step from the queue. A step either returns a result - * or throws. When the queue is empty the last step repeats. + * or throws. When the queue is empty the last step repeats. It records the argument of each call + * and every accepted result. */ private static final class ScriptedLoader implements FileDataReloader.Loader { final AtomicInteger calls = new AtomicInteger(); - final LinkedBlockingQueue> steps = new LinkedBlockingQueue<>(); - volatile Supplier last; + final List retryDelayArguments = Collections.synchronizedList(new ArrayList<>()); + final List accepted = Collections.synchronizedList(new ArrayList<>()); + final LinkedBlockingQueue steps = new LinkedBlockingQueue<>(); + volatile Step last; volatile CountDownLatch entered; volatile CountDownLatch release; @@ -81,7 +95,7 @@ ScriptedLoader then(LoadResult result) { ScriptedLoader thenFail(String message) { steps.add(() -> { - throw new UncheckedFileDataException(failure(message)); + throw failure(message); }); return this; } @@ -93,12 +107,24 @@ ScriptedLoader thenThrow(RuntimeException e) { return this; } + ScriptedLoader thenEmptied(Path path) { + return thenEmptied(path, SHORT); + } + + ScriptedLoader thenEmptied(Path path, Duration retryAfter) { + steps.add(() -> { + throw new EmptiedFileException(Collections.singletonList(path), retryAfter.toNanos()); + }); + return this; + } + @Override - public LoadResult load() throws FileDataException { + public LoadResult load(Duration retryDelay) throws FileDataException, EmptiedFileException { // Both latches are read before the entered signal, so a test that clears them after the // signal cannot change what this call does. CountDownLatch enteredLatch = entered; CountDownLatch releaseLatch = release; + retryDelayArguments.add(retryDelay); calls.incrementAndGet(); if (enteredLatch != null) { enteredLatch.countDown(); @@ -110,26 +136,18 @@ public LoadResult load() throws FileDataException { Thread.currentThread().interrupt(); } } - Supplier step = steps.poll(); + Step step = steps.poll(); if (step == null) { step = last; } else { last = step; } - try { - return step.get(); - } catch (UncheckedFileDataException e) { - throw e.wrapped; - } + return step.run(); } - } - - @SuppressWarnings("serial") - private static final class UncheckedFileDataException extends RuntimeException { - final FileDataException wrapped; - UncheckedFileDataException(FileDataException wrapped) { - this.wrapped = wrapped; + @Override + public void accepted(LoadResult result) { + accepted.add(result); } } @@ -539,4 +557,147 @@ public void unexpectedRuntimeExceptionFromLoaderIsLoggedAndWorkerSurvives() thro assertThat(logCapture.getMessageStrings(), hasItem(startsWith("ERROR:Unexpected error while reloading flag data:"))); } + + private static final Path EMPTIED = Paths.get("/overrides/emptied.yaml"); + private static final String READ_AGAIN_MESSAGE = "DEBUG:File " + EMPTIED + + " is empty, possibly while it is being rewritten; reading it again before applying it"; + + @Test + public void emptiedFileIsReadAgainAfterTheRetryDelayWithoutReportingAFailure() throws Exception { + LoadResult first = resultWithHash(1); + LoadResult rewritten = resultWithHash(2); + ScriptedLoader loader = new ScriptedLoader().then(first).thenEmptied(EMPTIED).then(rewritten); + RecordingHandler handler = new RecordingHandler(); + FileDataReloader r = reloader(loader, handler, Duration.ZERO, SHORT, true); + r.reloadNow(); + + // The load that finds the file emptied applies nothing. Only the retry, with no further + // trigger, loads again. + r.trigger(); + awaitSize(handler.applied, 2); + + assertEquals(first, handler.applied.get(0)); + assertEquals(rewritten, handler.applied.get(1)); + assertEquals(3, loader.calls.get()); + assertTrue(handler.errors.isEmpty()); + List messages = logCapture.getMessageStrings(); + assertThat(messages, hasItem(READ_AGAIN_MESSAGE)); + assertThat(messages, hasItem("DEBUG:Reading flag data again after a file was found empty")); + assertThat(messages, not(hasItem("DEBUG:Retrying flag data load after earlier failure"))); + for (String message : messages) { + assertFalse(message, message.startsWith("ERROR:")); + } + } + + @Test + public void holdIsRetriedAfterTheTimeTheLoaderAsksFor() throws Exception { + // The loader asks for a retry sooner than the retry delay, which would not come in this test. + LoadResult rewritten = resultWithHash(2); + ScriptedLoader loader = new ScriptedLoader().then(resultWithHash(1)).thenEmptied(EMPTIED, Duration.ofMillis(30)) + .then(rewritten); + RecordingHandler handler = new RecordingHandler(); + FileDataReloader r = reloader(loader, handler, Duration.ZERO, Duration.ofHours(1), true); + r.reloadNow(); + + r.trigger(); + awaitSize(handler.applied, 2); + assertEquals(rewritten, handler.applied.get(1)); + assertEquals(3, loader.calls.get()); + assertEquals(Duration.ofMillis(30).toNanos(), r.getLastRequestedRetryNanos()); + } + + @Test + public void failureAfterAHoldIsRetriedAfterTheRetryDelay() throws Exception { + // A hold asks for a retry soon. The failure that follows arms the normal retry delay, which does + // not come in this test. + ScriptedLoader loader = new ScriptedLoader().then(resultWithHash(1)).thenEmptied(EMPTIED, Duration.ofMillis(30)) + .thenFail("broken").then(resultWithHash(2)); + RecordingHandler handler = new RecordingHandler(); + FileDataReloader r = reloader(loader, handler, Duration.ZERO, Duration.ofHours(1), true); + r.reloadNow(); + + r.trigger(); + awaitSize(handler.errors, 1); + Thread.sleep(300); + assertEquals(3, loader.calls.get()); + assertEquals(1, handler.applied.size()); + assertEquals(Duration.ofHours(1).toNanos(), r.getLastRequestedRetryNanos()); + } + + @Test + public void loaderIsGivenTheRetryDelayOrZeroWithoutRetries() { + ScriptedLoader withRetries = new ScriptedLoader().then(resultWithHash(1)); + reloader(withRetries, new RecordingHandler(), Duration.ZERO, SHORT, true).reloadNow(); + assertEquals(Collections.singletonList(SHORT), withRetries.retryDelayArguments); + + // Without retries nothing would read a deferred file again, so the loader must not defer one. + ScriptedLoader withoutRetries = new ScriptedLoader().then(resultWithHash(1)); + reloader(withoutRetries, new RecordingHandler(), Duration.ZERO, Duration.ofMillis(-1), true).reloadNow(); + assertEquals(Collections.singletonList(Duration.ZERO), withoutRetries.retryDelayArguments); + } + + @Test + public void loaderIsToldOfEveryResultThatBecomesTheDataInEffect() throws Exception { + // An applied result and a result skipped as unchanged are the data in effect. A failure, a + // deferral and a result that the consumer rejects are not. + LoadResult first = resultWithHash(1); + LoadResult unchanged = resultWithHash(1); + LoadResult rejected = resultWithHash(2); + LoadResult last = resultWithHash(3); + ScriptedLoader loader = new ScriptedLoader().then(first).then(unchanged).thenFail("broken") + .thenEmptied(EMPTIED).then(rejected).then(last); + RecordingHandler handler = new RecordingHandler() { + @Override + public void apply(LoadResult r) { + if (r == rejected) { + throw new IllegalStateException("consumer failed"); + } + super.apply(r); + } + }; + FileDataReloader r = reloader(loader, handler, Duration.ZERO, Duration.ZERO, true); + r.reloadNow(); + for (int i = 2; i <= 6; i++) { + r.trigger(); + awaitAtLeast(loader.calls, i); + } + awaitSize(loader.accepted, 3); + + assertEquals(Arrays.asList(first, unchanged, last), loader.accepted); + assertEquals(Arrays.asList(first, last), handler.applied); + } + + @Test + public void failureAfterAReadAgainIsRetriedAsAFailure() throws Exception { + ScriptedLoader loader = new ScriptedLoader().then(resultWithHash(1)).thenEmptied(EMPTIED).thenFail("broken") + .then(resultWithHash(2)); + RecordingHandler handler = new RecordingHandler(); + FileDataReloader r = reloader(loader, handler, Duration.ZERO, SHORT, true); + r.reloadNow(); + + r.trigger(); + awaitSize(handler.applied, 2); + assertEquals(4, loader.calls.get()); + assertEquals(1, handler.errors.size()); + List messages = logCapture.getMessageStrings(); + assertThat(messages, hasItem("DEBUG:Reading flag data again after a file was found empty")); + assertThat(messages, hasItem("DEBUG:Retrying flag data load after earlier failure")); + } + + @Test + public void readingAnEmptiedFileAgainDoesNotEndARecovery() throws Exception { + // After a failure, the next success applies even when its content is unchanged. A read again + // in between is not that success. + LoadResult result = resultWithHash(1); + ScriptedLoader loader = new ScriptedLoader().then(result).thenFail("broken").thenEmptied(EMPTIED).then(result); + RecordingHandler handler = new RecordingHandler(); + FileDataReloader r = reloader(loader, handler, Duration.ZERO, SHORT, true); + r.reloadNow(); + + r.trigger(); + awaitSize(handler.applied, 2); + assertEquals(4, loader.calls.get()); + assertEquals(1, handler.errors.size()); + assertThat(logCapture.getMessageStrings(), hasItem(READ_AGAIN_MESSAGE)); + } } diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceInPlaceRewriteTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceInPlaceRewriteTest.java new file mode 100644 index 00000000..39898115 --- /dev/null +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceInPlaceRewriteTest.java @@ -0,0 +1,261 @@ +package com.launchdarkly.sdk.server.integrations; + +import com.launchdarkly.logging.LDLogLevel; +import com.launchdarkly.logging.LogCapture; +import com.launchdarkly.logging.Logs; +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.server.Components; +import com.launchdarkly.sdk.server.LDClient; +import com.launchdarkly.sdk.server.LDConfig; +import com.launchdarkly.sdk.server.TestComponents; +import com.launchdarkly.sdk.server.integrations.FileOverrides.ChangeDetection; +import com.launchdarkly.sdk.server.subsystems.ClientContext; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor; +import com.launchdarkly.sdk.server.subsystems.OverrideSource; +import com.launchdarkly.testhelpers.TempDir; + +import org.junit.After; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import static com.launchdarkly.sdk.server.DataModel.FEATURES; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * The file-based override source while its file is rewritten in place: the file is truncated, stays + * empty for a while, and then gets the new content. The overrides in effect must go from the old + * content to the new content without a moment in between without the override. + */ +@SuppressWarnings("javadoc") +public class FileOverrideSourceInPlaceRewriteTest { + private static final long WAIT_MILLIS = 10000; + private static final LDContext CONTEXT = LDContext.create("user-key"); + + // A watching source reads the file within the debounce delay of the truncation. The retry delay + // leaves a margin over the pause for a loaded machine. + private static final Duration WATCHING_PAUSE = Duration.ofMillis(300); + private static final Duration WATCHING_RETRY_DELAY = Duration.ofMillis(1500); + + // A polling source examines the file once per second. A pause longer than that makes sure that one + // examination finds the file empty, and the retry delay is longer than the pause by a margin. + private static final Duration POLL_INTERVAL = Duration.ofSeconds(1); + private static final Duration POLLING_PAUSE = Duration.ofMillis(1200); + private static final Duration POLLING_RETRY_DELAY = Duration.ofMillis(2500); + + private final LogCapture logCapture = Logs.capture(); + private final ClientContext context = TestComponents.clientContext("sdk-key", + new LDConfig.Builder().logging(Components.logging(logCapture).level(LDLogLevel.DEBUG)).build()); + private final List closeables = new ArrayList<>(); + + @After + public void close() throws Exception { + for (AutoCloseable c : closeables) { + c.close(); + } + } + + private static String doc(Path file, String value) { + return file.toString().endsWith(".yaml") ? "flagValues:\n flag: \"" + value + "\"\n" + : "{\"flagValues\":{\"flag\":\"" + value + "\"}}"; + } + + private static Path write(TempDir dir, String name, String value) throws Exception { + Path p = dir.getPath().resolve(name); + Files.write(p, doc(p, value).getBytes(StandardCharsets.UTF_8)); + return p; + } + + private static FileOverrideSourceImpl source(Path file, ChangeDetection mode, Duration retryDelay, + ClientContext context) { + return new FileOverrideSourceImpl(Collections.singletonList(file.toAbsolutePath()), + FileData.DuplicateKeysHandling.FAIL, mode, POLL_INTERVAL, retryDelay, + context.getBaseLogger().subLogger("FileOverrideSource"), FileDataWatcher::create); + } + + // The value of the override "flag" in a snapshot, or "none" when the snapshot does not have it. + private static String flagValue(Map> snapshot) { + ItemDescriptor item = snapshot.get(FEATURES.getName()).get("flag"); + if (item == null) { + return "none"; + } + return LDValue.parse(FEATURES.serialize(item)).get("variations").get(0).stringValue(); + } + + /** + * The writes a test makes to the file once the source has started. + */ + private interface Rewrites { + void run() throws Exception; + } + + // Starts the source on a file whose override has the value "a", makes the writes, and returns the + // value of the override in every snapshot the source supplies until one has the final value, and + // in any snapshot shortly after that. + private List appliedThroughRewrites(Path file, OverrideSource source, Rewrites rewrites, + String finalValue) throws Exception { + FileOverrideSourceTest.RecordingSink sink = new FileOverrideSourceTest.RecordingSink(); + closeables.add(source); + source.start(sink); + List applied = new ArrayList<>(); + applied.add(flagValue(sink.await())); + // Let the start of change detection settle. + Thread.sleep(300); + + rewrites.run(); + + long deadline = System.currentTimeMillis() + WAIT_MILLIS; + while (!applied.get(applied.size() - 1).equals(finalValue) && System.currentTimeMillis() < deadline) { + Map> snapshot = sink.snapshots.poll(100, TimeUnit.MILLISECONDS); + if (snapshot != null) { + applied.add(flagValue(snapshot)); + } + } + // A later snapshot would be wrong too. + Map> late = sink.snapshots.poll(500, TimeUnit.MILLISECONDS); + if (late != null) { + applied.add(flagValue(late)); + } + return applied; + } + + private void sourceKeepsTheOverrideThroughARewrite(String name, ChangeDetection mode, Duration pause, + Duration retryDelay) throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, name, "a"); + List applied = appliedThroughRewrites(file, source(file, mode, retryDelay, context), + () -> InPlaceWrites.rewrite(file, doc(file, "b"), pause), "b"); + assertEquals(Arrays.asList("a", "b"), applied); + } + } + + // The override must have a value in every snapshot, and the last one must be the final value. An + // intermediate value may or may not be applied. + private static void assertNeverWithoutTheOverride(List applied, String finalValue) { + assertEquals(applied.toString(), "a", applied.get(0)); + assertEquals(applied.toString(), finalValue, applied.get(applied.size() - 1)); + assertTrue(applied.toString(), !applied.contains("none")); + } + + @Test + public void watchingSourceKeepsTheOverrideThroughARewriteOfAJsonFile() throws Exception { + sourceKeepsTheOverrideThroughARewrite("overrides.json", ChangeDetection.WATCHING, WATCHING_PAUSE, + WATCHING_RETRY_DELAY); + } + + @Test + public void watchingSourceKeepsTheOverrideThroughARewriteOfAYamlFile() throws Exception { + sourceKeepsTheOverrideThroughARewrite("overrides.yaml", ChangeDetection.WATCHING, WATCHING_PAUSE, + WATCHING_RETRY_DELAY); + } + + @Test + public void pollingSourceKeepsTheOverrideThroughARewriteOfAJsonFile() throws Exception { + sourceKeepsTheOverrideThroughARewrite("overrides.json", ChangeDetection.POLLING, POLLING_PAUSE, + POLLING_RETRY_DELAY); + } + + @Test + public void pollingSourceKeepsTheOverrideThroughARewriteOfAYamlFile() throws Exception { + sourceKeepsTheOverrideThroughARewrite("overrides.yaml", ChangeDetection.POLLING, POLLING_PAUSE, + POLLING_RETRY_DELAY); + } + + @Test + public void watchingSourceKeepsTheOverrideThroughTwoQuickRewrites() throws Exception { + // The write of b and the second truncation fall in one debounce window, so no load reads b. + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", "a"); + List applied = appliedThroughRewrites(file, + source(file, ChangeDetection.WATCHING, WATCHING_RETRY_DELAY, context), () -> { + InPlaceWrites.rewrite(file, doc(file, "b"), Duration.ofMillis(150)); + InPlaceWrites.rewrite(file, doc(file, "c"), WATCHING_PAUSE); + }, "c"); + assertNeverWithoutTheOverride(applied, "c"); + } + } + + @Test + public void defaultSourceKeepsTheOverrideThroughRewritesASecondApart() throws Exception { + // The default source polls once per second and retries after one second. Two rewrites about a + // second apart can let a poll or a retry find the second truncation. + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", "a"); + List applied = appliedThroughRewrites(file, FileOverrides.source().filePaths(file).build(context), + () -> { + InPlaceWrites.rewrite(file, doc(file, "b"), WATCHING_PAUSE); + Thread.sleep(700); + InPlaceWrites.rewrite(file, doc(file, "c"), WATCHING_PAUSE); + }, "c"); + assertNeverWithoutTheOverride(applied, "c"); + } + } + + private void clientServesTheOverrideThroughARewrite(String name) throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, name, "a"); + TestDataV2 td = TestDataV2.synchronizer(); + td.update(td.flag("flag").valueForAll(LDValue.of("launchdarkly-value"))); + LDConfig config = new LDConfig.Builder() + .dataSystem(Components.dataSystem().custom().synchronizers(td) + .overrides(ctx -> source(file, ChangeDetection.WATCHING, WATCHING_RETRY_DELAY, ctx))) + .events(Components.noEvents()) + .logging(Components.logging(logCapture).level(LDLogLevel.DEBUG)) + .build(); + try (LDClient client = new LDClient("sdk-key", config)) { + // The distinct values the client serves, in order. + List served = Collections.synchronizedList(new ArrayList<>()); + AtomicBoolean stop = new AtomicBoolean(false); + Thread sampler = new Thread(() -> { + while (!stop.get()) { + String value = client.stringVariation("flag", CONTEXT, "default"); + if (served.isEmpty() || !served.get(served.size() - 1).equals(value)) { + served.add(value); + } + try { + Thread.sleep(1); + } catch (InterruptedException e) { + return; + } + } + }); + sampler.start(); + try { + Thread.sleep(300); + InPlaceWrites.rewrite(file, doc(file, "b"), WATCHING_PAUSE); + long deadline = System.currentTimeMillis() + WAIT_MILLIS; + while (!served.contains("b") && System.currentTimeMillis() < deadline) { + Thread.sleep(10); + } + Thread.sleep(500); + } finally { + stop.set(true); + sampler.join(); + } + assertEquals(Arrays.asList("a", "b"), served); + } + } + } + + @Test + public void clientServesTheOverrideThroughARewriteOfAJsonFile() throws Exception { + clientServesTheOverrideThroughARewrite("overrides.json"); + } + + @Test + public void clientServesTheOverrideThroughARewriteOfAYamlFile() throws Exception { + clientServesTheOverrideThroughARewrite("overrides.yaml"); + } +} diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceTest.java new file mode 100644 index 00000000..de95ee6e --- /dev/null +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceTest.java @@ -0,0 +1,584 @@ +package com.launchdarkly.sdk.server.integrations; + +import com.launchdarkly.logging.LDLogLevel; +import com.launchdarkly.logging.LogCapture; +import com.launchdarkly.logging.Logs; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.server.Components; +import com.launchdarkly.sdk.server.LDConfig; +import com.launchdarkly.sdk.server.TestComponents; +import com.launchdarkly.sdk.server.integrations.FileOverrides.ChangeDetection; +import com.launchdarkly.sdk.server.subsystems.ClientContext; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.DataKind; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.KeyedItems; +import com.launchdarkly.sdk.server.subsystems.OverrideSink; +import com.launchdarkly.sdk.server.subsystems.OverrideSource; +import com.launchdarkly.testhelpers.TempDir; + +import org.junit.After; +import org.junit.Test; + +import java.io.IOException; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import static com.launchdarkly.sdk.server.DataModel.FEATURES; +import static com.launchdarkly.sdk.server.DataModel.SEGMENTS; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.startsWith; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@SuppressWarnings("javadoc") +public class FileOverrideSourceTest { + private static final long WAIT_MILLIS = 10000; + + private final LogCapture logCapture = Logs.capture(); + private final ClientContext context = TestComponents.clientContext("sdk-key", + new LDConfig.Builder().logging(Components.logging(logCapture).level(LDLogLevel.DEBUG)).build()); + private final List sources = new ArrayList<>(); + + @After + public void closeSources() throws Exception { + for (OverrideSource s : sources) { + s.close(); + } + } + + /** + * Records every snapshot the source supplies, as a map of kind name to key to item. + */ + static final class RecordingSink implements OverrideSink { + final BlockingQueue>> snapshots = new LinkedBlockingQueue<>(); + + @Override + public void setOverrides(Iterable>> data) { + Map> snapshot = new HashMap<>(); + snapshot.put(FEATURES.getName(), new HashMap<>()); + snapshot.put(SEGMENTS.getName(), new HashMap<>()); + for (Map.Entry> kind : data) { + Map items = snapshot.computeIfAbsent(kind.getKey().getName(), k -> new HashMap<>()); + for (Map.Entry item : kind.getValue().getItems()) { + items.put(item.getKey(), item.getValue()); + } + } + snapshots.add(snapshot); + } + + Map> await() throws InterruptedException { + Map> s = snapshots.poll(WAIT_MILLIS, TimeUnit.MILLISECONDS); + if (s == null) { + throw new AssertionError("timed out waiting for an override snapshot"); + } + return s; + } + + void assertNoSnapshot(long millis) throws InterruptedException { + assertNull(snapshots.poll(millis, TimeUnit.MILLISECONDS)); + } + } + + private OverrideSource start(FileOverrideSourceBuilder builder, RecordingSink sink) { + OverrideSource source = builder.build(context); + sources.add(source); + source.start(sink); + return source; + } + + private static String flagValue(Map> snapshot, String key) { + ItemDescriptor item = snapshot.get(FEATURES.getName()).get(key); + if (item == null) { + return null; + } + LDValue json = LDValue.parse(FEATURES.serialize(item)); + return json.get("variations").get(0).stringValue(); + } + + private static Path write(TempDir dir, String name, String contents) throws Exception { + Path p = dir.getPath().resolve(name); + Files.write(p, contents.getBytes()); + return p; + } + + private static String docWith(String key, String value) { + return "{\"flagValues\":{\"" + key + "\":\"" + value + "\"}}"; + } + + @Test + public void buildRequiresFilePaths() { + try { + FileOverrides.source().build(context); + fail("expected exception"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("no file paths")); + } + } + + @Test + public void buildRequiresChangeDetectionMode() { + try { + FileOverrides.source().filePaths("x.json").changeDetection(null).build(context); + fail("expected exception"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("change detection")); + } + } + + @Test + public void buildRequiresPollInterval() { + try { + FileOverrides.source().filePaths("x.json").pollInterval(null).build(context); + fail("expected exception"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("poll interval")); + } + } + + @Test + public void buildRejectsWatchingAPathOnAnotherFileSystem() throws Exception { + // The default watch service cannot watch such a path. Rejecting it here reports the + // configuration error when the client is created, instead of a failure when the source starts. + try (TempDir dir = TempDir.create(); FileSystem zip = newZipFileSystem(dir)) { + Path file = zip.getPath("/overrides.json"); + try { + FileOverrides.source().filePaths(file).changeDetection(ChangeDetection.WATCHING).build(context); + fail("expected exception"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("can watch only files on the default file system")); + } + } + } + + @Test + public void pollingLoadsAPathOnAnotherFileSystem() throws Exception { + try (TempDir dir = TempDir.create(); FileSystem zip = newZipFileSystem(dir)) { + Path file = zip.getPath("/overrides.json"); + Files.write(file, docWith("flag", "value").getBytes(StandardCharsets.UTF_8)); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file).changeDetection(ChangeDetection.POLLING), sink); + + assertEquals("value", flagValue(sink.await(), "flag")); + } + } + + private static FileSystem newZipFileSystem(TempDir dir) throws Exception { + URI uri = URI.create("jar:" + dir.getPath().resolve("overrides.zip").toUri()); + return FileSystems.newFileSystem(uri, Collections.singletonMap("create", "true")); + } + + @Test + public void nullDuplicateKeysHandlingSelectsFail() throws Exception { + try (TempDir dir = TempDir.create()) { + Path a = write(dir, "a.json", docWith("flag", "a")); + Path b = write(dir, "b.json", docWith("flag", "b")); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(a, b).duplicateKeysHandling(null), sink); + sink.assertNoSnapshot(200); + assertThat(logCapture.getMessageStrings(), hasItem(allOf(startsWith("ERROR:Unable to load flags: "), + containsString("in features, key \"flag\" was already defined"), containsString(b.toString())))); + } + } + + @Test + public void repeatedPathIsDroppedWithAWarning() throws Exception { + // Under the default fail handling the same file listed twice would make every one of its + // keys a duplicate of itself. The repeat is ignored and the file loads once. + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "a.json", docWith("flag", "value")); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file, file), sink); + + assertEquals("value", flagValue(sink.await(), "flag")); + assertThat(logCapture.getMessageStrings(), + hasItem("WARN:File paths list the same file more than once; the repeats are ignored")); + } + } + + @Test + public void initialLoadCompletesBeforeStartReturns() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.json", + "{\"flags\":{\"full\":{\"key\":\"full\",\"version\":7,\"on\":true,\"variations\":[\"x\"],\"fallthrough\":{\"variation\":0}}}," + + "\"flagValues\":{\"simple\":\"value\"}," + + "\"segments\":{\"seg\":{\"key\":\"seg\",\"version\":3,\"included\":[\"u\"]}}}"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file), sink); + + Map> snapshot = sink.snapshots.poll(); + assertNotNull("the initial snapshot must be supplied before start returns", snapshot); + assertEquals(7, snapshot.get(FEATURES.getName()).get("full").getVersion()); + assertEquals("value", flagValue(snapshot, "simple")); + // A value-only entry is a flag served by fallthrough with version 0. + ItemDescriptor simple = snapshot.get(FEATURES.getName()).get("simple"); + assertEquals(0, simple.getVersion()); + LDValue simpleJson = LDValue.parse(FEATURES.serialize(simple)); + assertEquals(LDValue.of(true), simpleJson.get("on")); + assertEquals(LDValue.of(0), simpleJson.get("fallthrough").get("variation")); + assertEquals(LDValue.ofNull(), simpleJson.get("offVariation")); + assertEquals(3, snapshot.get(SEGMENTS.getName()).get("seg").getVersion()); + assertThat(logCapture.getMessageStrings(), hasItem( + "INFO:Flag overrides in effect: 2 flags, 1 segment (" + file + ": 2 flags, 1 segment)")); + } + } + + @Test + public void loadsYamlDocument() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", "flagValues:\n yaml-flag: \"override-value\"\n"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file), sink); + assertEquals("override-value", flagValue(sink.await(), "yaml-flag")); + } + } + + @Test + public void mergesFilesInConfiguredOrderWithIgnoreHandling() throws Exception { + try (TempDir dir = TempDir.create()) { + Path first = write(dir, "first.json", docWith("shared", "first")); + Path second = write(dir, "second.json", "{\"flagValues\":{\"shared\":\"second\",\"only-second\":\"x\"}}"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(first, second) + .duplicateKeysHandling(FileData.DuplicateKeysHandling.IGNORE), sink); + + Map> snapshot = sink.await(); + assertEquals("first", flagValue(snapshot, "shared")); + assertEquals("x", flagValue(snapshot, "only-second")); + assertThat(logCapture.getMessageStrings(), hasItem( + "INFO:Flag overrides in effect: 2 flags (" + first + ": 1 flag; " + second + ": 1 flag)")); + } + } + + @Test + public void duplicateKeysFailTheLoadByDefault() throws Exception { + try (TempDir dir = TempDir.create()) { + Path a = write(dir, "a.json", docWith("flag", "a")); + Path b = write(dir, "b.json", docWith("flag", "b")); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(a, b), sink); + sink.assertNoSnapshot(200); + } + } + + @Test + public void missingFileContributesNoEntriesAndIsNotAnError() throws Exception { + try (TempDir dir = TempDir.create()) { + Path present = write(dir, "present.json", docWith("flag", "value")); + Path missing = dir.getPath().resolve("missing.json"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(present, missing), sink); + + Map> snapshot = sink.await(); + assertEquals("value", flagValue(snapshot, "flag")); + assertEquals(1, snapshot.get(FEATURES.getName()).size()); + for (String message : logCapture.getMessageStrings()) { + assertTrue("unexpected error log: " + message, !message.startsWith("ERROR:")); + } + assertThat(logCapture.getMessageStrings(), hasItem( + "INFO:Flag overrides in effect: 1 flag (" + present + ": 1 flag; " + missing + ": absent)")); + } + } + + @Test + public void startsWithNoFilesAndLogsNoneInEffect() throws Exception { + try (TempDir dir = TempDir.create()) { + Path missing = dir.getPath().resolve("missing.json"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(missing), sink); + + Map> snapshot = sink.await(); + assertTrue(snapshot.get(FEATURES.getName()).isEmpty()); + assertThat(logCapture.getMessageStrings(), hasItem("INFO:Flag overrides: none in effect (" + missing + ": absent)")); + } + } + + @Test + public void emptyDocumentLogsNoEntries() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "empty.json", "{}"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file), sink); + sink.await(); + assertThat(logCapture.getMessageStrings(), hasItem("INFO:Flag overrides: none in effect (" + file + ": no entries)")); + } + } + + @Test + public void emptyYamlFileAtStartupHasNoEntries() throws Exception { + for (String contents : new String[] { "", " \n\n" }) { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", contents); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file), sink); + assertTrue(sink.await().get(FEATURES.getName()).isEmpty()); + assertThat(logCapture.getMessageStrings(), hasItem("INFO:Flag overrides: none in effect (" + file + ": no entries)")); + } + } + } + + @Test + public void emptyJsonFileAtStartupFailsToLoad() throws Exception { + for (String name : new String[] { "overrides.json", "overrides" }) { + for (String contents : new String[] { "", " \n\t\n" }) { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, name, contents); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file), sink); + sink.assertNoSnapshot(200); + assertThat(logCapture.getMessageStrings(), hasItem("ERROR:Unable to load flags: file " + file + + ": the file has no content, which is not a JSON document")); + } + } + } + } + + @Test + public void malformedFileAtStartupSuppliesNothingAndLogsError() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "bad.json", "{\"flagValues\""); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file), sink); + sink.assertNoSnapshot(200); + assertThat(logCapture.getMessageStrings(), hasItem(startsWith("ERROR:Unable to load flags:"))); + } + } + + private void reloadsOnChange(ChangeDetection mode) throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.json", docWith("flag", "first")); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file).changeDetection(mode).pollInterval(Duration.ofSeconds(1)), sink); + assertEquals("first", flagValue(sink.await(), "flag")); + + // A changed file is reloaded. + Files.write(file, docWith("flag", "second").getBytes()); + assertEquals("second", flagValue(sink.await(), "flag")); + + // A malformed edit keeps the last good overrides and supplies nothing. + Files.write(file, "{\"flagValues\"".getBytes()); + sink.assertNoSnapshot(1500); + assertThat(logCapture.getMessageStrings(), hasItem(startsWith("ERROR:Unable to load flags:"))); + + // A later good edit is applied. + Files.write(file, docWith("flag", "third").getBytes()); + assertEquals("third", flagValue(sink.await(), "flag")); + + // Deleting the file removes its overrides. + Files.delete(file); + assertTrue(sink.await().get(FEATURES.getName()).isEmpty()); + + // Recreating the file brings them back. + Files.write(file, docWith("flag", "fourth").getBytes()); + assertEquals("fourth", flagValue(sink.await(), "flag")); + } + } + + @Test + public void pollingModeReloadsOnChange() throws Exception { + reloadsOnChange(ChangeDetection.POLLING); + } + + @Test + public void watchingModeReloadsOnChange() throws Exception { + reloadsOnChange(ChangeDetection.WATCHING); + } + + @Test + public void watchingFallsBackToPollingWhenTheWatcherCannotBeCreated() throws Exception { + // Without change detection a later edit would never take effect. The source polls instead, at + // no less than the minimum poll interval, which watching mode otherwise ignores. + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.json", docWith("flag", "first")); + FileOverrideSourceImpl source = new FileOverrideSourceImpl( + Collections.singletonList(file.toAbsolutePath()), FileData.DuplicateKeysHandling.FAIL, + ChangeDetection.WATCHING, Duration.ofMillis(10), FileDataReloader.DEFAULT_RETRY_DELAY, + context.getBaseLogger().subLogger("FileOverrideSource"), + (paths, logger) -> { + throw new IOException("watch service unavailable"); + }); + sources.add(source); + RecordingSink sink = new RecordingSink(); + source.start(sink); + assertEquals("first", flagValue(sink.await(), "flag")); + + Files.write(file, docWith("flag", "second").getBytes()); + assertEquals("second", flagValue(sink.await(), "flag")); + assertThat(logCapture.getMessageStrings(), hasItem(allOf( + startsWith("ERROR:Unable to watch override files; polling them instead every PT1S"), + containsString("watch service unavailable")))); + } + } + + @Test + public void fileThatDoesNotExistYetTakesEffectWhenItAppears() throws Exception { + for (ChangeDetection mode : ChangeDetection.values()) { + try (TempDir dir = TempDir.create()) { + Path file = dir.getPath().resolve("later.json"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file).changeDetection(mode), sink); + assertTrue(sink.await().get(FEATURES.getName()).isEmpty()); + + Files.write(file, docWith("flag", "appeared").getBytes()); + assertEquals(mode.toString(), "appeared", flagValue(sink.await(), "flag")); + } + } + } + + @Test + public void pollIntervalBelowMinimumIsRaisedWithWarning() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.json", "{}"); + RecordingSink sink = new RecordingSink(); + OverrideSource source = start(FileOverrides.source().filePaths(file).pollInterval(Duration.ofMillis(10)), sink); + assertEquals(FileOverrides.MINIMUM_POLL_INTERVAL, ((FileOverrideSourceImpl) source).getPollInterval()); + assertThat(logCapture.getMessageStrings(), hasItem( + "WARN:Poll interval PT0.01S is below the minimum; using PT1S")); + } + } + + @Test + public void pollIntervalAtOrAboveMinimumIsKept() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.json", "{}"); + OverrideSource source = start(FileOverrides.source().filePaths(file).pollInterval(Duration.ofSeconds(3)), + new RecordingSink()); + assertEquals(Duration.ofSeconds(3), ((FileOverrideSourceImpl) source).getPollInterval()); + OverrideSource defaulted = start(FileOverrides.source().filePaths(file), new RecordingSink()); + assertEquals(FileOverrides.DEFAULT_POLL_INTERVAL, ((FileOverrideSourceImpl) defaulted).getPollInterval()); + } + } + + @Test + public void failedLoadIsRetriedWithoutAFileChange() throws Exception { + // Watching mode delivers no notification while the file is untouched, so only the source's own + // retry can attempt the load again. + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "bad.json", "{\"flagValues\""); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file).changeDetection(ChangeDetection.WATCHING), sink); + assertThat(logCapture.getMessageStrings(), hasItem(startsWith("ERROR:Unable to load flags:"))); + + long deadline = System.currentTimeMillis() + WAIT_MILLIS; + while (!logCapture.getMessageStrings().contains("DEBUG:Retrying flag data load after earlier failure")) { + if (System.currentTimeMillis() > deadline) { + fail("the failed load was not retried"); + } + Thread.sleep(50); + } + sink.assertNoSnapshot(100); + } + } + + @Test + public void watchingModeIgnoresPollIntervalMinimum() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.json", "{}"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file).changeDetection(ChangeDetection.WATCHING) + .pollInterval(Duration.ofMillis(10)), sink); + for (String message : logCapture.getMessageStrings()) { + assertTrue(message, !message.startsWith("WARN:Poll interval")); + } + } + } + + private static int liveThreadsNamed(String prefix) { + int count = 0; + for (Thread t : Thread.getAllStackTraces().keySet()) { + if (t.isAlive() && t.getName().startsWith(prefix)) { + count++; + } + } + return count; + } + + private static void awaitLiveThreadsNamed(String prefix, int expected) throws InterruptedException { + long deadline = System.currentTimeMillis() + WAIT_MILLIS; + while (liveThreadsNamed(prefix) != expected) { + if (System.currentTimeMillis() > deadline) { + fail("expected " + expected + " live threads named " + prefix + " but found " + liveThreadsNamed(prefix)); + } + Thread.sleep(20); + } + } + + private void closeReleasesThreads(ChangeDetection mode, String threadPrefix) throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.json", docWith("flag", "first")); + int baseline = liveThreadsNamed(threadPrefix); + RecordingSink sink = new RecordingSink(); + OverrideSource source = start(FileOverrides.source().filePaths(file).changeDetection(mode) + .pollInterval(Duration.ofSeconds(1)), sink); + sink.await(); + awaitLiveThreadsNamed(threadPrefix, baseline + 1); + + source.close(); + source.close(); + + awaitLiveThreadsNamed(threadPrefix, baseline); + Files.write(file, docWith("flag", "second").getBytes()); + sink.assertNoSnapshot(1500); + } + } + + @Test + public void closeIsIdempotentAndStopsThePoller() throws Exception { + closeReleasesThreads(ChangeDetection.POLLING, "LaunchDarkly-FileDataPoller"); + } + + @Test + public void closeIsIdempotentAndStopsTheWatcher() throws Exception { + closeReleasesThreads(ChangeDetection.WATCHING, "LaunchDarkly-FileDataWatcher"); + } + + @Test + public void countsTextFormatsCounts() { + assertEquals("1 flag", FileOverrideSourceImpl.countsText(1, 0)); + assertEquals("2 flags, 1 segment", FileOverrideSourceImpl.countsText(2, 1)); + assertEquals("3 segments", FileOverrideSourceImpl.countsText(0, 3)); + assertEquals("", FileOverrideSourceImpl.countsText(0, 0)); + } + + @Test + public void relativePathIsResolvedAgainstWorkingDirectory() throws Exception { + Path relative = java.nio.file.Paths.get("no-such-dir-for-override-test", "overrides.json"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(relative.toString()), sink); + assertTrue(sink.await().get(FEATURES.getName()).isEmpty()); + assertThat(logCapture.getMessageStrings(), hasItem(containsString(relative.toAbsolutePath().normalize().toString()))); + } + + @Test + public void loggerUsesSubLoggerName() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.json", "{}"); + start(FileOverrides.source().filePaths(file), new RecordingSink()); + boolean found = false; + for (LogCapture.Message m : logCapture.getMessages()) { + if (m.getLevel() == LDLogLevel.INFO && m.getLoggerName().endsWith("FileOverrideSource")) { + found = true; + } + } + assertTrue(found); + } + } +} diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/InPlaceWrites.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/InPlaceWrites.java new file mode 100644 index 00000000..69273d79 --- /dev/null +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/InPlaceWrites.java @@ -0,0 +1,49 @@ +package com.launchdarkly.sdk.server.integrations; + +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; + +/** + * Rewrites files in place the way many applications and editors save them: the file is truncated + * when it is opened for writing, and the new content is written through the same open stream. A + * reader in between finds the file empty. The tests use these writes, not an atomic replacement, so + * that the code under test must handle a file that is being rewritten. + */ +@SuppressWarnings("javadoc") +final class InPlaceWrites { + /** + * Work done while the file is empty, between the truncation and the write. + */ + interface WhileEmpty { + void run() throws Exception; + } + + private InPlaceWrites() {} + + /** + * Truncates the file, pauses, and then writes the new content. + */ + static void rewrite(Path path, String content, Duration pause) throws Exception { + rewrite(path, content, () -> Thread.sleep(pause.toMillis())); + } + + /** + * Truncates the file, runs the given work while the file is empty, and then writes the new content. + */ + static void rewrite(Path path, String content, WhileEmpty whileEmpty) throws Exception { + try (OutputStream out = Files.newOutputStream(path)) { + whileEmpty.run(); + out.write(content.getBytes(StandardCharsets.UTF_8)); + } + } + + /** + * Truncates the file and leaves it empty. + */ + static void truncate(Path path) throws Exception { + rewrite(path, "", () -> { }); + } +} diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/OverrideFileLoaderTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/OverrideFileLoaderTest.java index 2c00651c..cdd8665f 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/OverrideFileLoaderTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/OverrideFileLoaderTest.java @@ -1,6 +1,7 @@ package com.launchdarkly.sdk.server.integrations; import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.server.integrations.FileDataReloader.EmptiedFileException; import com.launchdarkly.sdk.server.integrations.FileDataSourceParsing.FileDataException; import com.launchdarkly.sdk.server.integrations.OverrideFileLoader.FileSummary; import com.launchdarkly.sdk.server.integrations.OverrideFileLoader.LoadResult; @@ -12,13 +13,17 @@ import org.junit.Test; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; import static com.launchdarkly.sdk.server.DataModel.FEATURES; import static com.launchdarkly.sdk.server.DataModel.SEGMENTS; @@ -27,6 +32,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -69,7 +75,7 @@ public void loadsFlagsFlagValuesAndSegmentsKeepingDocumentVersions() throws Exce "{\"flags\":{\"f1\":{\"key\":\"f1\",\"version\":7,\"on\":true,\"variations\":[true],\"fallthrough\":{\"variation\":0}}}," + "\"flagValues\":{\"f2\":\"x\"}," + "\"segments\":{\"s1\":{\"key\":\"s1\",\"version\":9}}}"); - LoadResult result = loader(file).load(); + LoadResult result = loader(file).load(Duration.ZERO); Map> data = toMap(result); assertEquals(7, data.get(FEATURES).get("f1").getVersion()); @@ -90,7 +96,7 @@ public void loadsFlagsFlagValuesAndSegmentsKeepingDocumentVersions() throws Exce public void valueOnlyEntryBecomesFallthroughFlagServingTheValue() throws Exception { try (TempDir dir = TempDir.create()) { Path file = write(dir, "overrides.json", "{\"flagValues\":{\"f2\":\"x\"}}"); - LoadResult result = loader(file).load(); + LoadResult result = loader(file).load(Duration.ZERO); ItemDescriptor f2 = toMap(result).get(FEATURES).get("f2"); assertEquals(0, f2.getVersion()); LDValue flag = json(FEATURES, f2); @@ -104,10 +110,10 @@ public void valueOnlyEntryBecomesFallthroughFlagServingTheValue() throws Excepti } @Test - public void yamlIsAutoDetected() throws Exception { + public void yamlFileIsParsedAsYaml() throws Exception { try (TempDir dir = TempDir.create()) { Path file = write(dir, "overrides.yaml", "flagValues:\n yaml-flag: \"override-value\"\n"); - LoadResult result = loader(file).load(); + LoadResult result = loader(file).load(Duration.ZERO); assertEquals(LDValue.buildArray().add("override-value").build(), json(FEATURES, toMap(result).get(FEATURES).get("yaml-flag")).get("variations")); } @@ -119,7 +125,7 @@ public void missingFileContributesNothing() throws Exception { Path present = write(dir, "present.json", "{\"flagValues\":{\"flag\":\"value\"}}"); Path missing = dir.getPath().resolve("missing.json"); Path segments = write(dir, "segments.json", "{\"segments\":{\"s1\":{\"key\":\"s1\"}}}"); - LoadResult result = loader(present, missing, segments).load(); + LoadResult result = loader(present, missing, segments).load(Duration.ZERO); Map> data = toMap(result); assertEquals(1, data.get(FEATURES).size()); @@ -139,7 +145,7 @@ public void missingFileContributesNothing() throws Exception { @Test public void allFilesMissingProducesEmptyResult() throws Exception { try (TempDir dir = TempDir.create()) { - LoadResult result = loader(dir.getPath().resolve("a.json"), dir.getPath().resolve("b.json")).load(); + LoadResult result = loader(dir.getPath().resolve("a.json"), dir.getPath().resolve("b.json")).load(Duration.ZERO); assertEquals(0, result.getFlagCount()); assertEquals(0, result.getSegmentCount()); assertFalse(result.getData().iterator().hasNext()); @@ -154,7 +160,7 @@ public void unreadableFileFailsTheLoad() throws Exception { Path directory = dir.getPath().resolve("dir.json"); Files.createDirectory(directory); try { - loader(directory).load(); + loader(directory).load(Duration.ZERO); fail("expected exception"); } catch (FileDataException e) { assertThat(e.getMessage(), containsString(directory.toString())); @@ -170,7 +176,7 @@ public void malformedDocumentFailsTheLoadWithFileAttribution() throws Exception Path good = write(dir, "good.json", "{\"flagValues\":{\"a\":1}}"); Path bad = write(dir, "bad.json", "{\"flagValues\""); try { - loader(good, bad).load(); + loader(good, bad).load(Duration.ZERO); fail("expected exception"); } catch (FileDataException e) { assertThat(e.getMessage(), containsString(bad.toString())); @@ -185,7 +191,7 @@ public void modelRejectedDocumentIsAFileDataError() throws Exception { try (TempDir dir = TempDir.create()) { Path file = write(dir, "bad-model.json", "{\"flags\":{\"f1\":{\"key\":\"f1\",\"rules\":5}}}"); try { - loader(file).load(); + loader(file).load(Duration.ZERO); fail("expected exception"); } catch (FileDataException e) { assertThat(e.getMessage(), containsString(file.toString())); @@ -201,7 +207,7 @@ public void duplicateKeysFailByDefaultAcrossFilesAndWithinAFile() throws Excepti Path a = write(dir, "a.json", "{\"flagValues\":{\"flag\":\"a\"}}"); Path b = write(dir, "b.json", "{\"flagValues\":{\"flag\":\"b\"}}"); try { - loader(a, b).load(); + loader(a, b).load(Duration.ZERO); fail("expected exception"); } catch (FileDataException e) { assertThat(e.getMessage(), containsString("in features, key \"flag\" was already defined")); @@ -211,7 +217,7 @@ public void duplicateKeysFailByDefaultAcrossFilesAndWithinAFile() throws Excepti Path both = write(dir, "both.json", "{\"flags\":{\"x\":{\"key\":\"x\"}},\"flagValues\":{\"x\":true}}"); try { - loader(both).load(); + loader(both).load(Duration.ZERO); fail("expected exception"); } catch (FileDataException e) { assertThat(e.getMessage(), containsString("key \"x\" was already defined")); @@ -220,7 +226,7 @@ public void duplicateKeysFailByDefaultAcrossFilesAndWithinAFile() throws Excepti Path seg1 = write(dir, "seg1.json", "{\"segments\":{\"s\":{\"key\":\"s\"}}}"); Path seg2 = write(dir, "seg2.json", "{\"segments\":{\"s\":{\"key\":\"s\"}}}"); try { - loader(seg1, seg2).load(); + loader(seg1, seg2).load(Duration.ZERO); fail("expected exception"); } catch (FileDataException e) { assertThat(e.getMessage(), containsString("in segments, key \"s\" was already defined")); @@ -234,7 +240,7 @@ public void ignoreHandlingKeepsFirstFileAndCountsOnlyKeptEntries() throws Except Path first = write(dir, "first.json", "{\"flagValues\":{\"shared\":\"first\"}}"); Path second = write(dir, "second.json", "{\"flagValues\":{\"shared\":\"second\",\"only-second\":\"x\"}}"); OverrideFileLoader loader = new OverrideFileLoader(Arrays.asList(first, second), FileData.DuplicateKeysHandling.IGNORE); - LoadResult result = loader.load(); + LoadResult result = loader.load(Duration.ZERO); Map flags = toMap(result).get(FEATURES); assertEquals(LDValue.buildArray().add("first").build(), json(FEATURES, flags.get("shared")).get("variations")); @@ -251,17 +257,17 @@ public void contentHashReflectsRawContentOfPresentFiles() throws Exception { Path file = write(dir, "overrides.json", "{\"flagValues\":{\"a\":1}}"); Path missing = dir.getPath().resolve("missing.json"); OverrideFileLoader loader = loader(file, missing); - byte[] first = loader.load().getContentHash(); - byte[] second = loader.load().getContentHash(); + byte[] first = loader.load(Duration.ZERO).getContentHash(); + byte[] second = loader.load(Duration.ZERO).getContentHash(); assertArrayEquals(first, second); Files.write(file, "{\"flagValues\":{\"a\":2}}".getBytes()); - byte[] third = loader.load().getContentHash(); + byte[] third = loader.load(Duration.ZERO).getContentHash(); assertThat(Arrays.equals(first, third), is(false)); // A file that appears changes the digest. Files.write(missing, "{}".getBytes()); - byte[] fourth = loader.load().getContentHash(); + byte[] fourth = loader.load(Duration.ZERO).getContentHash(); assertThat(Arrays.equals(third, fourth), is(false)); } } @@ -270,7 +276,7 @@ public void contentHashReflectsRawContentOfPresentFiles() throws Exception { public void emptyDocumentContributesNothing() throws Exception { try (TempDir dir = TempDir.create()) { Path file = write(dir, "empty.json", "{}"); - LoadResult result = loader(file).load(); + LoadResult result = loader(file).load(Duration.ZERO); assertEquals(0, result.getFlagCount()); assertTrue(result.getFiles().get(0).isPresent()); assertThat(result.getFiles().get(0).getFlags(), equalTo(0)); @@ -284,8 +290,1218 @@ public void loaderReadsFilesInConfiguredOrder() throws Exception { Path b = write(dir, "b.json", "{\"flagValues\":{\"k\":\"b\"}}"); OverrideFileLoader loader = new OverrideFileLoader(Arrays.asList(b, a), FileData.DuplicateKeysHandling.IGNORE); assertEquals(LDValue.buildArray().add("b").build(), - json(FEATURES, toMap(loader.load()).get(FEATURES).get("k")).get("variations")); - assertEquals(Collections.singletonList(b).get(0), loader.load().getFiles().get(0).getPath()); + json(FEATURES, toMap(loader.load(Duration.ZERO)).get(FEATURES).get("k")).get("variations")); + assertEquals(Collections.singletonList(b).get(0), loader.load(Duration.ZERO).getFiles().get(0).getPath()); + } + } + + private static final String YAML_DOC = "flagValues:\n flag: \"value\"\n"; + private static final String JSON_DOC = "{\"flagValues\":{\"flag\":\"value\"}}"; + private static final Duration RETRY = Duration.ofSeconds(1); + + /** + * A monotonic clock that the test moves. It starts near the largest value, so that elapsed times + * cross the overflow of the clock's readings. + */ + private static final class TestClock { + final AtomicLong nanos = new AtomicLong(Long.MAX_VALUE - RETRY.toNanos() / 2); + + void advance(double retryDelays) { + nanos.addAndGet((long) (retryDelays * RETRY.toNanos())); + } + } + + private static OverrideFileLoader loader(TestClock clock, Path... paths) { + return new OverrideFileLoader(Arrays.asList(paths), FileData.DuplicateKeysHandling.FAIL, clock.nanos::get); + } + + // Loads and applies the result, as the reloader does with a result it applies. + private static LoadResult loadAndAccept(OverrideFileLoader loader) throws Exception { + LoadResult result = loader.load(RETRY); + loader.accepted(result); + return result; + } + + private static void assertNoContentFailure(OverrideFileLoader loader, Path file) throws Exception { + try { + loader.load(RETRY); + fail("expected exception for " + file); + } catch (FileDataException e) { + assertEquals("file " + file + ": the file has no content, which is not a JSON document", e.getMessage()); + assertNull(e.getCause()); + } + } + + // Asserts that a load holds back the given files and returns how long it asks the reloader to + // wait, in retry delays. + private static double assertHeld(OverrideFileLoader loader, Path... files) throws Exception { + try { + loader.load(RETRY); + fail("expected the empty files " + Arrays.asList(files) + " to be held back"); + return 0; + } catch (EmptiedFileException e) { + assertEquals(Arrays.asList(files), e.getPaths()); + return (double) e.getRetryAfterNanos() / RETRY.toNanos(); + } + } + + // Writes the file again, in place, with a later modification time than the previous write. File + // system timestamps can be coarse, so it waits until the time differs. It writes other content first, + // so that the file is modified even when the new content is what it already holds. + private static void writeAgain(Path file, String contents) throws Exception { + long before = Files.getLastModifiedTime(file).toMillis(); + long deadline = System.currentTimeMillis() + 5000; + do { + Thread.sleep(20); + InPlaceWrites.rewrite(file, "x", () -> { }); + InPlaceWrites.rewrite(file, contents, () -> { }); + } while (Files.getLastModifiedTime(file).toMillis() == before && System.currentTimeMillis() < deadline); + assertThat(Files.getLastModifiedTime(file).toMillis(), not(equalTo(before))); + } + + private static void truncateAgain(Path file) throws Exception { + writeAgain(file, ""); + } + + @Test + public void formatIsChosenByTheFileNameExtension() throws Exception { + try (TempDir dir = TempDir.create()) { + for (String name : new String[] { "a.yaml", "a.yml", "a.YAML", "a.Yml" }) { + assertEquals(name, 1, loader(write(dir, name, YAML_DOC)).load(Duration.ZERO).getFlagCount()); + } + // A YAML file can hold most JSON documents. The YAML parser reads YAML 1.1, which rejects some + // JSON, such as indentation with tabs or the escape \/. + assertEquals(1, loader(write(dir, "json.yaml", JSON_DOC)).load(Duration.ZERO).getFlagCount()); + for (String json : new String[] { "{\n\t\"flagValues\": {\"flag\": \"v\"}\n}", "{\"flagValues\":{\"flag\":\"a\\/b\"}}" }) { + Path file = write(dir, "strict.yaml", json); + try { + loader(file).load(Duration.ZERO); + fail("expected the YAML parser to reject " + json); + } catch (FileDataException e) { + assertThat(e.getMessage(), containsString(file + ": unable to parse YAML")); + } + } + // Any other file is JSON, whatever its content. + for (String name : new String[] { "b.json", "b.txt", "b", "b.yaml.bak" }) { + Path file = write(dir, name, YAML_DOC); + try { + loader(file).load(Duration.ZERO); + fail("expected exception for " + name); + } catch (FileDataException e) { + assertThat(e.getMessage(), containsString(file + ": cannot parse JSON")); + } + } + } + } + + @Test + public void emptyOrWhiteSpaceJsonFileFailsToParse() throws Exception { + try (TempDir dir = TempDir.create()) { + // A file with no extension or another extension is JSON, so it gets the same rule. + for (String name : new String[] { "empty.json", "empty", "empty.txt" }) { + for (String contents : new String[] { "", " \n\t\r\n" }) { + Path file = write(dir, name, contents); + assertNoContentFailure(loader(file), file); + } + } + } + } + + @Test + public void yamlFileWithNoDocumentHasNoEntries() throws Exception { + try (TempDir dir = TempDir.create()) { + for (String name : new String[] { "empty.yaml", "empty.yml" }) { + for (String contents : new String[] { "", " \n\n", "# only a comment\n", "---\n" }) { + LoadResult result = loader(write(dir, name, contents)).load(RETRY); + assertEquals(0, result.getFlagCount()); + assertTrue(result.getFiles().get(0).isPresent()); + assertFalse(contents, result.getFiles().get(0).hasContent()); + } + } + } + } + + @Test + public void documentWithNoEntriesHasContent() throws Exception { + try (TempDir dir = TempDir.create()) { + for (String name : new String[] { "empty.yaml", "empty.json" }) { + LoadResult result = loader(write(dir, name, "{}")).load(RETRY); + assertEquals(0, result.getFlagCount()); + assertTrue(name, result.getFiles().get(0).hasContent()); + } + } + } + + @Test + public void emptiedYamlFileIsHeldForExactlyTheRetryDelayAndThenHasNoEntries() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", YAML_DOC); + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, file); + assertTrue(loadAndAccept(loader).getFiles().get(0).hasContent()); + + InPlaceWrites.truncate(file); + assertHeld(loader, file); + clock.advance(0.95); + assertHeld(loader, file); + // The file has stayed empty without being written for the retry delay. + clock.advance(0.05); + assertEquals(0, loader.load(RETRY).getFlagCount()); + // It is not held back again until it changes. + assertEquals(0, loader.load(RETRY).getFlagCount()); + } + } + + @Test + public void emptiedJsonFileIsHeldForTheRetryDelayAndThenFailsToParse() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.json", JSON_DOC); + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, file); + loadAndAccept(loader); + + InPlaceWrites.truncate(file); + assertHeld(loader, file); + clock.advance(1); + assertNoContentFailure(loader, file); + assertNoContentFailure(loader, file); + } + } + + @Test + public void holdAsksForARetryWhenTheFirstDelayEnds() throws Exception { + try (TempDir dir = TempDir.create()) { + Path first = write(dir, "first.yaml", YAML_DOC); + Path second = write(dir, "second.yaml", "flagValues:\n other: \"value\"\n"); + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, first, second); + loadAndAccept(loader); + + // The retry comes when the delay ends, plus a tenth of the retry delay for a timer that fires a + // little early. + InPlaceWrites.truncate(first); + assertEquals(1.1, assertHeld(loader, first), 1e-9); + clock.advance(0.4); + assertEquals(0.7, assertHeld(loader, first), 1e-9); + // A second file found empty later does not delay the retry for the first. + InPlaceWrites.truncate(second); + clock.advance(0.2); + assertEquals(0.5, assertHeld(loader, first, second), 1e-9); + } + } + + @Test + public void holdAsksForARetryWhenTheBoundEnds() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", YAML_DOC); + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, file); + loadAndAccept(loader); + + InPlaceWrites.truncate(file); + assertHeld(loader, file); + for (int i = 0; i < 4; i++) { + clock.advance(0.9); + writeAgain(file, " "); + assertHeld(loader, file); + } + // 4.6 retry delays after the first hold, the file is written again. Its delay would end at 5.6, + // its bound at 5.0. + clock.advance(1.0); + writeAgain(file, " "); + assertEquals(0.5, assertHeld(loader, file), 1e-9); + } + } + + @Test + public void contentThatIsOnlyWhiteSpaceOrAByteOrderMarkCountsAsEmpty() throws Exception { + // A read can see only the byte order mark that an editor writes first. For JSON that is not a + // parse failure to report but an empty file to hold back. + for (String name : new String[] { "overrides.yaml", "overrides.json" }) { + try (TempDir dir = TempDir.create()) { + String doc = name.endsWith(".json") ? JSON_DOC : YAML_DOC; + Path file = write(dir, name, doc); + OverrideFileLoader loader = loader(new TestClock(), file); + for (String contents : new String[] { " \n\t\r\n", "\uFEFF", "\uFEFF\n" }) { + assertEquals(name, 1, loadAndAccept(loader).getFlagCount()); + InPlaceWrites.rewrite(file, contents, () -> { }); + assertHeld(loader, file); + InPlaceWrites.rewrite(file, doc, () -> { }); + } + } + } + } + + @Test + public void fileEmptiedAgainWithTheSameStateAfterContentStartsANewDelay() throws Exception { + // File system timestamps are coarse, so a file can be emptied again with the modification time + // and size that it had when it was last found empty. A load that read content in between shows + // that this is a new emptying, so its delay starts over. Nothing is applied in between: the test + // does not accept the result of the load that reads the content. + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", YAML_DOC); + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, file); + loadAndAccept(loader); + + InPlaceWrites.truncate(file); + java.nio.file.attribute.FileTime emptiedAt = Files.getLastModifiedTime(file); + assertHeld(loader, file); + clock.advance(0.8); + InPlaceWrites.rewrite(file, YAML_DOC, () -> { }); + assertEquals(1, loader.load(RETRY).getFlagCount()); + + clock.advance(0.4); + InPlaceWrites.truncate(file); + Files.setLastModifiedTime(file, emptiedAt); + assertHeld(loader, file); + // The new delay is still running a little later. + clock.advance(0.2); + assertHeld(loader, file); + } + } + + @Test + public void yamlWithOnlyACommentOrADocumentMarkerIsHeld() throws Exception { + // A read can see only the first line of a YAML file that is being written. + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", YAML_DOC); + OverrideFileLoader loader = loader(new TestClock(), file); + for (String contents : new String[] { "# overrides for the incident\n", "---\n" }) { + assertEquals(1, loadAndAccept(loader).getFlagCount()); + InPlaceWrites.rewrite(file, contents, () -> { }); + assertHeld(loader, file); + InPlaceWrites.rewrite(file, YAML_DOC, () -> { }); + } + } + } + + @Test + public void documentWithNoEntriesIsAppliedRightAway() throws Exception { + // {} is how to remove a file's overrides without deleting it. + try (TempDir dir = TempDir.create()) { + for (String name : new String[] { "overrides.yaml", "overrides.json" }) { + Path file = write(dir, name, name.endsWith(".json") ? JSON_DOC : YAML_DOC); + OverrideFileLoader loader = loader(new TestClock(), file); + loadAndAccept(loader); + + InPlaceWrites.rewrite(file, "{}", () -> { }); + assertEquals(name, 0, loader.load(RETRY).getFlagCount()); + } + } + } + + @Test + public void contentThatWasReadButNotAppliedDoesNotHoldTheFile() throws Exception { + // Only the data in effect counts. A load whose result was not applied, for example because + // another file failed to parse, does not make the file's later emptiness a rewrite. + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", YAML_DOC); + OverrideFileLoader loader = loader(new TestClock(), file); + loader.load(RETRY); + + InPlaceWrites.truncate(file); + assertEquals(0, loader.load(RETRY).getFlagCount()); + } + } + + @Test + public void fileThatHasNoContentFromTheStartIsNeverHeld() throws Exception { + for (String placeholderContents : new String[] { "", "# placeholder\n" }) { + try (TempDir dir = TempDir.create()) { + Path placeholder = write(dir, "placeholder.yaml", placeholderContents); + Path data = write(dir, "data.json", JSON_DOC); + OverrideFileLoader loader = loader(new TestClock(), placeholder, data); + for (int i = 0; i < 3; i++) { + assertEquals(1, loadAndAccept(loader).getFlagCount()); + } + truncateAgain(placeholder); + assertEquals(1, loadAndAccept(loader).getFlagCount()); + } + } + } + + @Test + public void fileWhoseEmptyStateIsInEffectIsNotHeldAgain() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", YAML_DOC); + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, file); + loadAndAccept(loader); + + InPlaceWrites.truncate(file); + assertHeld(loader, file); + clock.advance(1); + assertEquals(0, loadAndAccept(loader).getFlagCount()); + + // A new truncation of the file whose empty state is in effect is parsed right away. + writeAgain(file, " "); + assertEquals(0, loader.load(RETRY).getFlagCount()); + } + } + + @Test + public void fileThatIsDeletedAndCreatedEmptyIsParsed() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", YAML_DOC); + OverrideFileLoader loader = loader(new TestClock(), file); + loadAndAccept(loader); + + Files.delete(file); + assertFalse(loadAndAccept(loader).getFiles().get(0).isPresent()); + Files.write(file, new byte[0]); + LoadResult result = loader.load(RETRY); + assertTrue(result.getFiles().get(0).isPresent()); + assertEquals(0, result.getFlagCount()); + } + } + + @Test + public void newWriteStartsTheDelayAgain() throws Exception { + // A write and a new truncation that no load saw in between: the file is in a new rewrite, not one + // that has stayed empty. + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", YAML_DOC); + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, file); + loadAndAccept(loader); + + InPlaceWrites.truncate(file); + assertHeld(loader, file); + clock.advance(0.8); + writeAgain(file, YAML_DOC); + InPlaceWrites.truncate(file); + assertHeld(loader, file); + clock.advance(0.95); + assertHeld(loader, file); + clock.advance(0.05); + assertEquals(0, loader.load(RETRY).getFlagCount()); + } + } + + @Test + public void newWriteOfALinkedFileStartsTheDelayAgain() throws Exception { + // The state of a file that is configured through a symbolic link is that of the file it points + // to; the link itself does not change when the file is written. + try (TempDir dir = TempDir.create()) { + Path target = write(dir, "target.yaml", YAML_DOC); + Path link = dir.getPath().resolve("overrides.yaml"); + try { + Files.createSymbolicLink(link, target); + } catch (UnsupportedOperationException | java.io.IOException e) { + org.junit.Assume.assumeNoException("symbolic links are not available", e); + } + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, link); + loadAndAccept(loader); + + InPlaceWrites.truncate(target); + assertHeld(loader, link); + clock.advance(0.8); + truncateAgain(target); + assertHeld(loader, link); + clock.advance(0.95); + assertHeld(loader, link); + clock.advance(0.05); + assertEquals(0, loader.load(RETRY).getFlagCount()); + } + } + + @Test + public void fileThatKeepsBeingWrittenIsParsedAfterItsBound() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", YAML_DOC); + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, file); + loadAndAccept(loader); + + InPlaceWrites.truncate(file); + assertHeld(loader, file); + // Every load finds a new write, so the delay keeps starting again, up to the bound of five retry + // delays from the first hold. + for (int i = 0; i < 5; i++) { + clock.advance(0.9); + truncateAgain(file); + assertHeld(loader, file); + } + clock.advance(0.5); + truncateAgain(file); + assertEquals(0, loader.load(RETRY).getFlagCount()); + } + } + + @Test + public void filesRewrittenInTurnAreParsedAfterTheirBound() throws Exception { + // Each load finds one of the files empty and the other with content, so each file's own delay + // starts again whenever it is found empty. A read with content keeps the file's bound, so it + // still ends. + try (TempDir dir = TempDir.create()) { + Path a = write(dir, "a.yaml", YAML_DOC); + Path b = write(dir, "b.yaml", "flagValues:\n other: \"value\"\n"); + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, a, b); + loadAndAccept(loader); + + InPlaceWrites.truncate(a); + assertHeld(loader, a); + for (int i = 0; i < 9; i++) { + clock.advance(0.5); + Path emptied = i % 2 == 0 ? b : a; + Path written = i % 2 == 0 ? a : b; + writeAgain(written, written == a ? YAML_DOC : "flagValues:\n other: \"value\"\n"); + truncateAgain(emptied); + assertHeld(loader, emptied); + } + clock.advance(0.5); + writeAgain(b, "flagValues:\n other: \"value\"\n"); + truncateAgain(a); + assertEquals(1, loader.load(RETRY).getFlagCount()); + } + } + + @Test + public void loadThatHoldsNothingBackStartsTheBoundOver() throws Exception { + // A load that reads the file with content and holds nothing back starts the file's bound over, + // although nothing is applied here because the test does not accept the result. + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", YAML_DOC); + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, file); + loadAndAccept(loader); + + InPlaceWrites.truncate(file); + assertHeld(loader, file); + for (int i = 0; i < 5; i++) { + clock.advance(0.9); + truncateAgain(file); + assertHeld(loader, file); + } + writeAgain(file, YAML_DOC); + assertEquals(1, loader.load(RETRY).getFlagCount()); + truncateAgain(file); + clock.advance(0.5); + assertHeld(loader, file); + clock.advance(0.4); + assertHeld(loader, file); + } + } + + @Test + public void applyingDataStartsTheBoundOver() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", YAML_DOC); + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, file); + loadAndAccept(loader); + + InPlaceWrites.truncate(file); + assertHeld(loader, file); + for (int i = 0; i < 5; i++) { + clock.advance(0.9); + truncateAgain(file); + assertHeld(loader, file); + } + writeAgain(file, YAML_DOC); + assertEquals(1, loadAndAccept(loader).getFlagCount()); + truncateAgain(file); + clock.advance(0.5); + assertHeld(loader, file); + clock.advance(0.4); + assertHeld(loader, file); + } + } + + @Test + public void jsonFileThatStaysEmptyIsHeldBackAgainWhenRewrittenLater() throws Exception { + // A JSON file that stays empty is parsed after the delay and fails. That load holds nothing back, + // so the file's bound starts over, and when it is later rewritten in place, the new truncation is + // held back again instead of reported. + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.json", JSON_DOC); + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, file); + loadAndAccept(loader); + + InPlaceWrites.truncate(file); + assertHeld(loader, file); + clock.advance(1); + assertNoContentFailure(loader, file); + clock.advance(10); + assertNoContentFailure(loader, file); + + writeAgain(file, " "); + assertHeld(loader, file); + InPlaceWrites.rewrite(file, JSON_DOC, () -> { }); + assertEquals(1, loader.load(RETRY).getFlagCount()); + } + } + + @Test + public void failingOtherFileDoesNotRestartTheBound() throws Exception { + // a.yaml keeps being written while empty. Another file fails to load now and then, which must not + // give a.yaml a fresh bound. + try (TempDir dir = TempDir.create()) { + Path a = write(dir, "a.yaml", YAML_DOC); + String other = "{\"flagValues\":{\"other\":\"value\"}}"; + Path c = write(dir, "c.json", other); + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, a, c); + loadAndAccept(loader); + + InPlaceWrites.truncate(a); + assertHeld(loader, a); + for (int i = 1; i < 10; i++) { + clock.advance(0.5); + writeAgain(a, " "); + if (i % 2 == 1) { + InPlaceWrites.rewrite(c, "{bad", () -> { }); + try { + loader.load(RETRY); + fail("expected exception"); + } catch (FileDataException e) { + assertThat(e.getMessage(), containsString(c.toString())); + } + } else { + InPlaceWrites.rewrite(c, other, () -> { }); + assertHeld(loader, a); + } + } + clock.advance(0.5); + writeAgain(a, " "); + InPlaceWrites.rewrite(c, other, () -> { }); + LoadResult result = loader.load(RETRY); + assertEquals(1, result.getFlagCount()); + assertFalse(result.getFiles().get(0).hasContent()); + } + } + + // Loads the files and applies the result, as the reloader does, or reports that the load held a + // file back. + private static boolean loadApplies(OverrideFileLoader loader) throws Exception { + try { + loader.accepted(loader.load(RETRY)); + return true; + } catch (EmptiedFileException e) { + return false; + } + } + + @Test + public void filesEmptiedInStaggeredTurnsAreAppliedAfterTheBound() throws Exception { + // Two files stay empty and are truncated again in turn, so that whenever one of them has stayed + // empty for the whole delay, the other is still within its delay and holds the load back. A file + // that settles so does not get a fresh bound while the other is held back. + try (TempDir dir = TempDir.create()) { + Path a = write(dir, "a.yaml", YAML_DOC); + Path b = write(dir, "b.yaml", "flagValues:\n other: \"value\"\n"); + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, a, b); + loadAndAccept(loader); + + InPlaceWrites.truncate(a); + assertHeld(loader, a); + boolean applied = false; + for (int i = 1; i <= 14 && !applied; i++) { + clock.advance(0.9); + if (i == 1) { + InPlaceWrites.truncate(b); + } else { + truncateAgain(i % 2 == 1 ? b : a); + } + applied = loadApplies(loader); + if (!applied) { + // The retry that the hold arms, a little after the other file's delay is over. + clock.advance(0.2); + applied = loadApplies(loader); + clock.advance(-0.2); + } + } + assertTrue("the files were held back without end", applied); + } + } + + @Test + public void boundOfAFileHeldBackDuringAFailureStartsOverOnceItIsReadAgain() throws Exception { + // While b.json fails to load, a.yaml is rewritten once and its gap is held back. Later, b.json is + // fixed while a.yaml is in the middle of another rewrite, which is held back with a bound of its own. + try (TempDir dir = TempDir.create()) { + Path a = write(dir, "a.yaml", YAML_DOC); + String other = "{\"flagValues\":{\"other\":\"value\"}}"; + Path b = write(dir, "b.json", other); + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, a, b); + loadAndAccept(loader); + + InPlaceWrites.rewrite(b, "{bad", () -> { }); + InPlaceWrites.truncate(a); + assertLoadFailsFor(loader, b); + InPlaceWrites.rewrite(a, "flagValues:\n flag: \"other value\"\n", () -> { }); + assertLoadFailsFor(loader, b); + clock.advance(6); + assertLoadFailsFor(loader, b); + + InPlaceWrites.rewrite(b, other, () -> { }); + InPlaceWrites.truncate(a); + assertHeld(loader, a); + } + } + + @Test + public void fileFoundMissingStartsANewDelayWhenItIsCreatedEmptyAgain() throws Exception { + // While b.json fails, a.yaml is emptied, deleted, and created empty again with the state it had + // when it was first found empty, as within one tick of a coarse clock. Being missing ended its + // empty state, so it is held back with a new delay. + try (TempDir dir = TempDir.create()) { + Path a = write(dir, "a.yaml", YAML_DOC); + String other = "{\"flagValues\":{\"other\":\"value\"}}"; + Path b = write(dir, "b.json", other); + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, a, b); + loadAndAccept(loader); + + InPlaceWrites.rewrite(b, "{bad", () -> { }); + InPlaceWrites.truncate(a); + java.nio.file.attribute.FileTime emptiedAt = Files.getLastModifiedTime(a); + assertLoadFailsFor(loader, b); + clock.advance(0.5); + Files.delete(a); + assertLoadFailsFor(loader, b); + clock.advance(0.67); + Files.write(a, new byte[0]); + Files.setLastModifiedTime(a, emptiedAt); + assertLoadFailsFor(loader, b); + + InPlaceWrites.rewrite(b, other, () -> { }); + assertHeld(loader, a); + } + } + + @Test + public void fixOfAFileSavedInPlaceWithAnErrorIsHeldBack() throws Exception { + // An operator saves the file in place with a mistake, and saves the fix in place long after. The + // loads in between fail on the file, and the fix's own gap is held back. + for (String name : new String[] { "data.yaml", "data.json" }) { + try (TempDir dir = TempDir.create()) { + boolean yaml = name.endsWith(".yaml"); + Path file = write(dir, name, yaml ? YAML_DOC : JSON_DOC); + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, file); + loadAndAccept(loader); + + InPlaceWrites.truncate(file); + assertHeld(loader, file); + InPlaceWrites.rewrite(file, yaml ? "flagValues: [\n" : "{\"flagValues\": ", () -> { }); + assertLoadFailsFor(loader, file); + for (int i = 0; i < 6; i++) { + clock.advance(1); + assertLoadFailsFor(loader, file); + } + + InPlaceWrites.truncate(file); + assertHeld(loader, file); + InPlaceWrites.rewrite(file, yaml ? "flagValues:\n flag: \"fixed\"\n" : "{\"flagValues\":{\"flag\":\"fixed\"}}", + () -> { }); + assertEquals(name, 1, loader.load(RETRY).getFlagCount()); + } + } + } + + @Test + public void fileThatFailedToParseStartsANewDelayWhenItIsEmptiedAgain() throws Exception { + // Content that fails to parse also ends the file's empty state, so an emptying after it starts a + // new delay, even when a coarse clock gives it the state of the earlier emptying. + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", YAML_DOC); + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, file); + loadAndAccept(loader); + + InPlaceWrites.truncate(file); + java.nio.file.attribute.FileTime emptiedAt = Files.getLastModifiedTime(file); + assertHeld(loader, file); + clock.advance(0.5); + InPlaceWrites.rewrite(file, "flagValues: [\n", () -> { }); + assertLoadFailsFor(loader, file); + clock.advance(0.7); + InPlaceWrites.truncate(file); + Files.setLastModifiedTime(file, emptiedAt); + assertHeld(loader, file); + } + } + + @Test + public void fileThatFailsToReadStartsItsBoundOver() throws Exception { + // A load that fails because a file cannot be read holds nothing back, so it starts that file's + // bound over too. + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", YAML_DOC); + TestClock clock = new TestClock(); + AtomicLong unreadable = new AtomicLong(); + OverrideFileLoader loader = new OverrideFileLoader(Collections.singletonList(file), + FileData.DuplicateKeysHandling.FAIL, clock.nanos::get, path -> { + if (unreadable.get() != 0) { + throw new java.io.IOException("permission denied"); + } + return Files.readAllBytes(path); + }); + loadAndAccept(loader); + + InPlaceWrites.truncate(file); + assertHeld(loader, file); + unreadable.set(1); + for (int i = 0; i < 6; i++) { + clock.advance(1); + assertLoadFailsFor(loader, file); + } + unreadable.set(0); + writeAgain(file, " "); + assertHeld(loader, file); + } + } + + @Test + public void boundOfAFileAfterAFailingFileStartsOverOnceItIsReadAgain() throws Exception { + // c.json comes first. While it fails, a.yaml, which was held back, gets its content; the failing + // loads still read a.yaml, so its bound starts over, and the in-place rewrite of a.yaml that comes + // with the fix of c.json is held back. + try (TempDir dir = TempDir.create()) { + String other = "{\"flagValues\":{\"other\":\"value\"}}"; + Path c = write(dir, "c.json", other); + Path a = write(dir, "a.yaml", YAML_DOC); + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, c, a); + loadAndAccept(loader); + + InPlaceWrites.truncate(a); + assertHeld(loader, a); + InPlaceWrites.rewrite(c, "{bad", () -> { }); + InPlaceWrites.rewrite(a, YAML_DOC, () -> { }); + for (int i = 0; i < 6; i++) { + assertLoadFailsFor(loader, c); + clock.advance(1); + } + InPlaceWrites.rewrite(c, other, () -> { }); + InPlaceWrites.truncate(a); + assertHeld(loader, a); + } + } + + @Test + public void failingLoadReportsTheFirstFailure() throws Exception { + try (TempDir dir = TempDir.create()) { + Path first = write(dir, "first.json", "{bad"); + Path second = write(dir, "second.yaml", "flagValues: [\n"); + try { + loader(first, second).load(RETRY); + fail("expected exception"); + } catch (FileDataException e) { + assertThat(e.getMessage(), containsString(first + ": cannot parse JSON")); + } + } + } + + @Test + public void fileAfterADuplicateKeyIsReadSoItsBoundStartsOver() throws Exception { + // The entries are merged only after every file is read, so a duplicate key between a.yaml and + // b.yaml does not keep the loads from reading c.yaml, whose bound starts over while they fail. + try (TempDir dir = TempDir.create()) { + Path a = write(dir, "a.yaml", "flagValues:\n k: \"a\"\n"); + String bDoc = "flagValues:\n j: \"b\"\n"; + Path b = write(dir, "b.yaml", bDoc); + Path c = write(dir, "c.yaml", YAML_DOC); + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, a, b, c); + loadAndAccept(loader); + + InPlaceWrites.truncate(c); + assertHeld(loader, c); + InPlaceWrites.rewrite(b, "flagValues:\n j: \"b\"\n k: \"dup\"\n", () -> { }); + InPlaceWrites.rewrite(c, YAML_DOC, () -> { }); + for (int i = 0; i < 8; i++) { + clock.advance(1); + try { + loader.load(RETRY); + fail("expected exception"); + } catch (FileDataException e) { + assertThat(e.getMessage(), containsString("key \"k\" was already defined")); + } + } + InPlaceWrites.rewrite(b, bDoc, () -> { }); + truncateAgain(c); + assertHeld(loader, c); + } + } + + @Test + public void fixOfAFileSavedInPlaceWithADuplicateKeyIsHeldBack() throws Exception { + // As with a parse error: b.yaml is saved in place with a key that a.yaml defines, then the fix is + // saved in place several retry delays later. The fix's gap is held back. + try (TempDir dir = TempDir.create()) { + Path a = write(dir, "a.yaml", "flagValues:\n k: \"a\"\n"); + Path b = write(dir, "b.yaml", "flagValues:\n j: \"b\"\n"); + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, a, b); + loadAndAccept(loader); + + InPlaceWrites.truncate(b); + assertHeld(loader, b); + InPlaceWrites.rewrite(b, "flagValues:\n j: \"b\"\n k: \"dup\"\n", () -> { }); + for (int i = 0; i < 7; i++) { + try { + loader.load(RETRY); + fail("expected exception"); + } catch (FileDataException e) { + assertThat(e.getMessage(), containsString("key \"k\" was already defined")); + } + clock.advance(1); + } + InPlaceWrites.truncate(b); + assertHeld(loader, b); + } + } + + @Test + public void slowReadOfTheEmptiedFileItselfDoesNotShortenItsDelay() throws Exception { + // The emptied file's own read stalls for most of a retry delay, as on a network file system. Its + // delay counts from after its read. + try (TempDir dir = TempDir.create()) { + Path a = write(dir, "a.yaml", YAML_DOC); + TestClock clock = new TestClock(); + AtomicLong slowRead = new AtomicLong(); + OverrideFileLoader loader = new OverrideFileLoader(Collections.singletonList(a), + FileData.DuplicateKeysHandling.FAIL, clock.nanos::get, path -> { + byte[] raw = Files.readAllBytes(path); + if (slowRead.getAndSet(0) != 0) { + clock.advance(0.9); + } + return raw; + }); + loadAndAccept(loader); + + InPlaceWrites.truncate(a); + slowRead.set(1); + assertHeld(loader, a); + clock.advance(0.2); + assertHeld(loader, a); + } + } + + @Test + public void jsonDocumentThatIsNotAnObjectFailsToParse() throws Exception { + try (TempDir dir = TempDir.create()) { + for (String doc : new String[] { "null", "[]", "5", "\"x\"" }) { + Path file = write(dir, "overrides.json", doc); + try { + loader(file).load(Duration.ZERO); + fail("expected exception for " + doc); + } catch (FileDataException e) { + assertEquals("file " + file + ": the JSON document is not an object", e.getMessage()); + assertNull(e.getCause()); + } + } + } + } + + @Test + public void textEncodingIsDetectedFromAByteOrderMarkAndIsUtf8Otherwise() throws Exception { + // Applies to both formats. Without a byte order mark the text is UTF-8, whatever the platform's + // default encoding. + String[] names = { "overrides.json", "overrides.yaml" }; + java.nio.charset.Charset[] charsets = { StandardCharsets.UTF_8, StandardCharsets.UTF_16LE, + StandardCharsets.UTF_16BE }; + try (TempDir dir = TempDir.create()) { + for (String name : names) { + String doc = name.endsWith(".json") ? "{\"flagValues\":{\"flag\":\"caf\u00e9\"}}" + : "flagValues:\n flag: \"caf\u00e9\"\n"; + for (java.nio.charset.Charset charset : charsets) { + for (boolean bom : new boolean[] { true, false }) { + if (!bom && charset != StandardCharsets.UTF_8) { + continue; + } + Path file = dir.getPath().resolve(name); + Files.write(file, ((bom ? "\uFEFF" : "") + doc).getBytes(charset)); + LoadResult result = loader(file).load(Duration.ZERO); + String what = name + " " + charset + (bom ? " with a byte order mark" : ""); + assertEquals(what, LDValue.buildArray().add("caf\u00e9").build(), + json(FEATURES, toMap(result).get(FEATURES).get("flag")).get("variations")); + } + } + } + } + } + + @Test + public void utf16FileWithOnlyWhiteSpaceCountsAsEmpty() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", YAML_DOC); + OverrideFileLoader loader = loader(new TestClock(), file); + loadAndAccept(loader); + + Files.write(file, "\uFEFF \n".getBytes(StandardCharsets.UTF_16LE)); + assertHeld(loader, file); + } + } + + @Test + public void contentHashCoversPathsAndPresence() throws Exception { + // The format depends on the path, so the same bytes moved from one configured file to another + // can be different data. + try (TempDir dir = TempDir.create()) { + Path yaml = write(dir, "overrides.yaml", "{\"flagValues\": {\"flag\": on}}"); + Path json = dir.getPath().resolve("overrides.json"); + OverrideFileLoader loader = loader(yaml, json); + LoadResult before = loader.load(Duration.ZERO); + Files.move(yaml, json); + LoadResult after = loader.load(Duration.ZERO); + assertThat(Arrays.equals(before.getContentHash(), after.getContentHash()), is(false)); + assertEquals(LDValue.of(true), json(FEATURES, toMap(before).get(FEATURES).get("flag")).get("variations").get(0)); + assertEquals(LDValue.of("on"), json(FEATURES, toMap(after).get(FEATURES).get("flag")).get("variations").get(0)); + } + } + + // Loads the files and applies the result, as the reloader does, or reports that the load held a + // file back or failed. + private static boolean loadAppliesOrFails(OverrideFileLoader loader) throws Exception { + try { + loader.accepted(loader.load(RETRY)); + return true; + } catch (EmptiedFileException | FileDataException e) { + return false; + } + } + + @Test + public void loadThatHoldsAFileBackAndThenFailsDoesNotStartBoundsOver() throws Exception { + // a.yaml and b.yaml are rewritten in turn, so that every load holds one of them back, and c.json + // fails at every other load. Those loads hold a file back, so they start no bound over. + try (TempDir dir = TempDir.create()) { + Path a = write(dir, "a.yaml", YAML_DOC); + String bDoc = "flagValues:\n other: \"value\"\n"; + Path b = write(dir, "b.yaml", bDoc); + String cGood = "{\"flagValues\":{\"third\":\"value\"}}"; + Path c = write(dir, "c.json", cGood); + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, a, b, c); + loadAndAccept(loader); + + InPlaceWrites.truncate(a); + assertHeld(loader, a); + boolean applied = false; + for (int i = 1; i <= 12 && !applied; i++) { + clock.advance(0.5); + boolean odd = i % 2 == 1; + InPlaceWrites.rewrite(odd ? a : b, odd ? YAML_DOC : bDoc, () -> { }); + InPlaceWrites.truncate(odd ? b : a); + InPlaceWrites.rewrite(c, odd ? "{bad" : cGood, () -> { }); + applied = loadAppliesOrFails(loader); + } + assertTrue("the files were held back without end", applied); + } + } + + @Test + public void fileKeptWrittenWhileEmptyIsAppliedAfterItsBoundWhileAnEarlierFileFails() throws Exception { + // c.json comes first and fails now and then, while a.yaml keeps being written while empty. The + // failing loads still read a.yaml and hold it back, so they do not start its bound over. + try (TempDir dir = TempDir.create()) { + String other = "{\"flagValues\":{\"other\":\"value\"}}"; + Path c = write(dir, "c.json", other); + Path a = write(dir, "a.yaml", YAML_DOC); + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, c, a); + loadAndAccept(loader); + + InPlaceWrites.truncate(a); + assertHeld(loader, a); + for (int i = 1; i <= 5; i++) { + clock.advance(0.9); + writeAgain(a, i % 2 == 0 ? " " : " "); + if (i % 2 == 1) { + InPlaceWrites.rewrite(c, "{bad", () -> { }); + assertLoadFailsFor(loader, c); + } else { + InPlaceWrites.rewrite(c, other, () -> { }); + assertHeld(loader, a); + } + } + clock.advance(0.9); + writeAgain(a, " "); + InPlaceWrites.rewrite(c, other, () -> { }); + assertEquals(1, loader.load(RETRY).getFlagCount()); + } + } + + @Test + public void holdAsksForTheRetryFromTheEndOfTheLoad() throws Exception { + // The read of the file after the held one takes longer than what is left of the held file's + // delay, so the retry is due right away, after the margin. + try (TempDir dir = TempDir.create()) { + Path a = write(dir, "a.yaml", YAML_DOC); + Path b = write(dir, "b.yaml", "flagValues:\n other: \"value\"\n"); + TestClock clock = new TestClock(); + AtomicLong slowReadOf = new AtomicLong(); + OverrideFileLoader loader = new OverrideFileLoader(Arrays.asList(a, b), FileData.DuplicateKeysHandling.FAIL, + clock.nanos::get, path -> { + byte[] raw = Files.readAllBytes(path); + if (path.equals(b) && slowReadOf.getAndSet(0) != 0) { + clock.advance(0.1); + } + return raw; + }); + loadAndAccept(loader); + + InPlaceWrites.truncate(a); + assertEquals(1.1, assertHeld(loader, a), 1e-9); + clock.advance(0.95); + slowReadOf.set(1); + assertEquals(0.1, assertHeld(loader, a), 1e-9); + } + } + + private static void assertLoadFailsFor(OverrideFileLoader loader, Path file) throws Exception { + try { + loader.load(RETRY); + fail("expected the load to fail"); + } catch (FileDataException e) { + assertThat(e.getMessage(), containsString(file.toString())); + } + } + + @Test + public void anotherFilesBoundDoesNotCutShortARewrite() throws Exception { + // b.yaml keeps being written while empty, and reaches its bound while a.yaml is in the middle of an + // ordinary rewrite. a.yaml is still held back. + try (TempDir dir = TempDir.create()) { + Path a = write(dir, "a.yaml", YAML_DOC); + Path b = write(dir, "b.yaml", "flagValues:\n other: \"value\"\n"); + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, a, b); + loadAndAccept(loader); + + InPlaceWrites.truncate(b); + assertHeld(loader, b); + for (int i = 1; i <= 9; i++) { + clock.advance(0.5); + writeAgain(b, " "); + if (i == 9) { + InPlaceWrites.truncate(a); + assertHeld(loader, a, b); + } else { + assertHeld(loader, b); + } + } + // b's bound is over now, but a has been empty for half a retry delay only. + clock.advance(0.5); + writeAgain(b, " "); + assertHeld(loader, a); + + clock.advance(0.25); + InPlaceWrites.rewrite(a, YAML_DOC, () -> { }); + LoadResult result = loader.load(RETRY); + assertEquals(1, result.getFlagCount()); + assertTrue(result.getFiles().get(0).hasContent()); + } + } + + @Test + public void holdAsksForTheRestOfTheDelayAfterASlowReadOfAnotherFile() throws Exception { + // a.yaml is held back, and the read of b.yaml after it takes half a retry delay. The retry comes + // when a.yaml's delay is over, counted from when the load asks for it. + try (TempDir dir = TempDir.create()) { + Path a = write(dir, "a.yaml", YAML_DOC); + Path b = write(dir, "b.yaml", "flagValues:\n other: \"value\"\n"); + TestClock clock = new TestClock(); + AtomicLong slowReadOf = new AtomicLong(); + OverrideFileLoader loader = new OverrideFileLoader(Arrays.asList(a, b), FileData.DuplicateKeysHandling.FAIL, + clock.nanos::get, path -> { + byte[] raw = Files.readAllBytes(path); + if (path.equals(b) && slowReadOf.getAndSet(0) != 0) { + clock.advance(0.5); + } + return raw; + }); + loadAndAccept(loader); + + InPlaceWrites.truncate(a); + slowReadOf.set(1); + assertEquals(0.6, assertHeld(loader, a), 1e-9); + } + } + + @Test + public void slowReadOfAnotherFileDoesNotShortenTheDelay() throws Exception { + // The read of b.yaml, which comes first, takes most of a retry delay. a.yaml's delay counts from + // when a.yaml was read. + try (TempDir dir = TempDir.create()) { + Path a = write(dir, "a.yaml", YAML_DOC); + Path b = write(dir, "b.yaml", "flagValues:\n other: \"value\"\n"); + TestClock clock = new TestClock(); + AtomicLong slowReadOf = new AtomicLong(); + OverrideFileLoader loader = new OverrideFileLoader(Arrays.asList(b, a), FileData.DuplicateKeysHandling.FAIL, + clock.nanos::get, path -> { + byte[] raw = Files.readAllBytes(path); + if (path.equals(b) && slowReadOf.getAndSet(0) != 0) { + clock.advance(0.94); + } + return raw; + }); + loadAndAccept(loader); + + InPlaceWrites.truncate(a); + slowReadOf.set(1); + assertHeld(loader, a); + clock.advance(0.1); + assertHeld(loader, a); + } + } + + @Test + public void withoutRetriesAnEmptiedFileIsParsedRightAway() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", YAML_DOC); + OverrideFileLoader loader = loader(new TestClock(), file); + loader.accepted(loader.load(Duration.ZERO)); + + InPlaceWrites.truncate(file); + assertEquals(0, loader.load(Duration.ZERO).getFlagCount()); + } + } + + @Test + public void filesEmptiedTogetherAreHeldFromTheSameLoad() throws Exception { + // A load that holds back one file still reads the others, so each emptied file starts its delay + // in that load and all of them are parsed after one retry delay, not one retry delay per file. + try (TempDir dir = TempDir.create()) { + Path first = write(dir, "first.yaml", YAML_DOC); + Path second = write(dir, "second.yaml", "flagValues:\n other: \"value\"\n"); + TestClock clock = new TestClock(); + OverrideFileLoader loader = loader(clock, first, second); + assertEquals(2, loadAndAccept(loader).getFlagCount()); + + InPlaceWrites.truncate(first); + InPlaceWrites.truncate(second); + assertHeld(loader, first, second); + clock.advance(1); + assertEquals(0, loader.load(RETRY).getFlagCount()); + } + } + + @Test + public void failureOfAnotherFileFailsALoadThatHoldsAFile() throws Exception { + try (TempDir dir = TempDir.create()) { + Path first = write(dir, "first.yaml", YAML_DOC); + Path second = write(dir, "second.json", JSON_DOC.replace("flag", "other")); + OverrideFileLoader loader = loader(new TestClock(), first, second); + loadAndAccept(loader); + + InPlaceWrites.truncate(first); + InPlaceWrites.rewrite(second, "{\"flagValues\"", () -> { }); + try { + loader.load(RETRY); + fail("expected exception"); + } catch (FileDataException e) { + assertThat(e.getMessage(), containsString(second + ": cannot parse JSON")); + } } } } diff --git a/lib/sdk/server/src/test/resources/override-vectors/vectors.json b/lib/sdk/server/src/test/resources/override-vectors/vectors.json new file mode 100644 index 00000000..82dc62da --- /dev/null +++ b/lib/sdk/server/src/test/resources/override-vectors/vectors.json @@ -0,0 +1,316 @@ +{ + "schemaVersion": "0.4.0", + "vectors": [ + { + "description": "An override takes precedence over LaunchDarkly data for the same flag key", + "group": "precedence", + "launchDarklyData": { + "initialized": true, + "flags": { + "flag-a": { + "key": "flag-a", + "version": 7, + "on": true, + "variations": ["ld-value"], + "fallthrough": { "variation": 0 }, + "offVariation": 0, + "salt": "salt-a" + } + }, + "segments": {} + }, + "overrides": { + "flagValues": { "flag-a": "override-value" } + }, + "evaluate": { + "flagKey": "flag-a", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "override-value", + "variationIndex": 0, + "reason": { "kind": "FALLTHROUGH", "overrideAffected": true }, + "summaryOverrideAffected": true + } + }, + { + "description": "An overridden flag is served when the SDK has not initialized from LaunchDarkly", + "group": "uninitialized", + "launchDarklyData": { + "initialized": false, + "flags": {}, + "segments": {} + }, + "overrides": { + "flagValues": { "flag-a": "override-value" } + }, + "evaluate": { + "flagKey": "flag-a", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "override-value", + "variationIndex": 0, + "reason": { "kind": "FALLTHROUGH", "overrideAffected": true }, + "summaryOverrideAffected": true + } + }, + { + "description": "A flag absent from the override layer is evaluated from LaunchDarkly data and is not marked", + "group": "passthrough", + "launchDarklyData": { + "initialized": true, + "flags": { + "flag-b": { + "key": "flag-b", + "version": 3, + "on": true, + "variations": ["off-value", "ld-value"], + "fallthrough": { "variation": 1 }, + "offVariation": 0, + "salt": "salt-b" + } + }, + "segments": {} + }, + "overrides": { + "flagValues": { "flag-a": "override-value" } + }, + "evaluate": { + "flagKey": "flag-b", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "ld-value", + "variationIndex": 1, + "reason": { "kind": "FALLTHROUGH" }, + "summaryOverrideAffected": false + } + }, + { + "description": "A flag absent from the override layer still short-circuits when the SDK has not initialized", + "group": "passthrough", + "launchDarklyData": { + "initialized": false, + "flags": {}, + "segments": {} + }, + "overrides": { + "flagValues": { "flag-a": "override-value" } + }, + "evaluate": { + "flagKey": "flag-b", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "fallback", + "variationIndex": null, + "reason": { "kind": "ERROR", "errorKind": "CLIENT_NOT_READY" } + } + }, + { + "description": "A full flag override with a matching targeting rule serves the rule's variation and is marked", + "group": "full-evaluation", + "launchDarklyData": { + "initialized": true, + "flags": {}, + "segments": {} + }, + "overrides": { + "flags": { + "flag-c": { + "key": "flag-c", + "version": 1, + "on": true, + "variations": ["default-value", "eu-value"], + "fallthrough": { "variation": 0 }, + "offVariation": 0, + "rules": [ + { + "id": "rule-eu", + "variation": 1, + "clauses": [ + { + "contextKind": "user", + "attribute": "region", + "op": "in", + "values": ["eu"], + "negate": false + } + ] + } + ], + "salt": "salt-c" + } + } + }, + "evaluate": { + "flagKey": "flag-c", + "context": { "kind": "user", "key": "user-1", "region": "eu" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "eu-value", + "variationIndex": 1, + "reason": { "kind": "RULE_MATCH", "ruleIndex": 0, "ruleId": "rule-eu", "overrideAffected": true }, + "summaryOverrideAffected": true + } + }, + { + "description": "An override flag resolves a referenced segment through the merged view while the SDK is uninitialized", + "group": "merged-view", + "launchDarklyData": { + "initialized": false, + "flags": {}, + "segments": {} + }, + "overrides": { + "flags": { + "flag-d": { + "key": "flag-d", + "version": 1, + "on": true, + "variations": ["default-value", "member-value"], + "fallthrough": { "variation": 0 }, + "offVariation": 0, + "rules": [ + { + "id": "rule-seg", + "variation": 1, + "clauses": [ + { + "attribute": "", + "op": "segmentMatch", + "values": ["seg-1"], + "negate": false + } + ] + } + ], + "salt": "salt-d" + } + }, + "segments": { + "seg-1": { + "key": "seg-1", + "version": 1, + "included": ["user-1"], + "excluded": [], + "salt": "seg-salt-1" + } + } + }, + "evaluate": { + "flagKey": "flag-d", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "member-value", + "variationIndex": 1, + "reason": { "kind": "RULE_MATCH", "ruleIndex": 0, "ruleId": "rule-seg", "overrideAffected": true }, + "summaryOverrideAffected": true + } + }, + { + "description": "A flag is marked overrideAffected when it matches via an overridden segment", + "group": "marking", + "launchDarklyData": { + "initialized": true, + "flags": { + "flag-f": { + "key": "flag-f", + "version": 4, + "on": true, + "variations": ["default-value", "member-value"], + "fallthrough": { "variation": 0 }, + "offVariation": 0, + "rules": [ + { + "id": "rule-seg", + "variation": 1, + "clauses": [ + { + "attribute": "", + "op": "segmentMatch", + "values": ["seg-2"], + "negate": false + } + ] + } + ], + "salt": "salt-f" + } + }, + "segments": { + "seg-2": { + "key": "seg-2", + "version": 1, + "included": [], + "excluded": [], + "salt": "seg-salt-2" + } + } + }, + "overrides": { + "segments": { + "seg-2": { + "key": "seg-2", + "version": 2, + "included": ["user-1"], + "excluded": [], + "salt": "seg-salt-2" + } + } + }, + "evaluate": { + "flagKey": "flag-f", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "member-value", + "variationIndex": 1, + "reason": { "kind": "RULE_MATCH", "ruleIndex": 0, "ruleId": "rule-seg", "overrideAffected": true }, + "summaryOverrideAffected": true + } + }, + { + "description": "A malformed override flag returns the caller default with an error reason and is marked overrideAffected", + "group": "error-handling", + "launchDarklyData": { + "initialized": true, + "flags": {}, + "segments": {} + }, + "overrides": { + "flags": { + "flag-e": { + "key": "flag-e", + "version": 1, + "on": true, + "variations": ["only-value"], + "fallthrough": { "variation": 5 }, + "offVariation": 0, + "salt": "salt-e" + } + } + }, + "evaluate": { + "flagKey": "flag-e", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "fallback", + "variationIndex": null, + "reason": { "kind": "ERROR", "errorKind": "MALFORMED_FLAG", "overrideAffected": true }, + "summaryOverrideAffected": true + } + } + ] +}