From e66b2c84680797c6d613d46b7206ddabdcc7f20f Mon Sep 17 00:00:00 2001 From: Hongwei Date: Mon, 2 Nov 2026 16:37:53 +0100 Subject: [PATCH 1/2] fix(dynamic-entity): drop the memoized definition lists when definitions change DynamicEntityHelper rebuilds its definitions map from NewStyle.getDynamicEntities, which keeps its list in Redis for dynamicEntity.cache.ttl.seconds. forgetDefinitions now also deletes the memoized getDynamicEntities lists, once the request's transaction has committed, so the rebuilt map reads the current rows and a new entity's record endpoints answer immediately instead of after the map's time-to-live. This node's map is still dropped at once for reads later in the same request. --- .../entity/helper/DynamicEntityHelper.scala | 20 +++- .../main/scala/code/api/util/NewStyle.scala | 12 +++ .../code/api/cache/CacheKeyFormatTest.scala | 19 ++++ .../DefinitionsCacheInvalidationTest.scala | 101 ++++++++++++++++++ 4 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 obp-api/src/test/scala/code/api/dynamic/entity/helper/DefinitionsCacheInvalidationTest.scala diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala b/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala index 9043c19a11..03169dfbdc 100644 --- a/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala +++ b/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala @@ -214,8 +214,26 @@ object DynamicEntityHelper { @volatile private var definitionsMapCache: Option[(Long, Map[(String, String), DynamicEntityInfo])] = None @volatile private var definitionsToken: AnyRef = new Object - /** Forget the kept definitions map, so the next use reads the definitions again. */ + /** + * Forget the kept definitions map and the memoized definition lists, so the next use reads the definitions + * again. The map is rebuilt from NewStyle.getDynamicEntities, which keeps its list in Redis for + * dynamicEntity.cache.ttl.seconds: forgetting only the map rebuilt it from the list as it was before the + * change, and kept that for the map's whole time-to-live. + * + * This node's map is dropped at once, for reads later in the same request. The map and the lists are dropped + * again once the request's transaction commits (at once outside a transaction): until then the change is not + * visible to other connections, so anything read in between holds the old definitions, and clearing the + * shared lists earlier would achieve nothing. Other nodes' maps still follow their own time-to-live. + */ def forgetDefinitions(): Unit = { + dropDefinitionsMap() + code.api.util.http4s.RequestScopeConnection.afterCommit { + dropDefinitionsMap() + NewStyle.function.invalidateDynamicEntityCache() + } + } + + private def dropDefinitionsMap(): Unit = { definitionsToken = new Object definitionsMapCache = None } diff --git a/obp-api/src/main/scala/code/api/util/NewStyle.scala b/obp-api/src/main/scala/code/api/util/NewStyle.scala index e2194dbdf8..6c7e2644b4 100644 --- a/obp-api/src/main/scala/code/api/util/NewStyle.scala +++ b/obp-api/src/main/scala/code/api/util/NewStyle.scala @@ -3716,6 +3716,18 @@ object NewStyle extends MdcLoggable{ } } + /** + * Drop every memoized `getDynamicEntities(...)` and `getDynamicEntitiesByUserId(...)` list after a + * definition is created, updated or deleted, so the change shows in the next read instead of after + * `dynamicEntity.cache.ttl.seconds`. As with `invalidateMethodRoutingCache`, the Redis memoize key embeds + * the method name, so one pattern delete clears every argument variant, for every node reading Redis. + * Called through DynamicEntityHelper.forgetDefinitions once the change has committed. No-op / logged when + * Redis is unavailable. + */ + def invalidateDynamicEntityCache(): Unit = { + Redis.deleteKeysByPattern("*getDynamicEntities*") + } + def getDynamicEntitiesByUserId(userId: String): List[DynamicEntityT] = { import scala.concurrent.duration._ diff --git a/obp-api/src/test/scala/code/api/cache/CacheKeyFormatTest.scala b/obp-api/src/test/scala/code/api/cache/CacheKeyFormatTest.scala index 0a97c8326b..0fc3972f98 100644 --- a/obp-api/src/test/scala/code/api/cache/CacheKeyFormatTest.scala +++ b/obp-api/src/test/scala/code/api/cache/CacheKeyFormatTest.scala @@ -48,6 +48,7 @@ import scala.jdk.CollectionConverters._ * is in there somewhere. That is not enough for the thing that actually depends on this format: * * NewStyle.scala:3306 Redis.deleteKeysByPattern("*getMethodRoutings*") + * NewStyle.scala Redis.deleteKeysByPattern("*getDynamicEntities*") * Caching.scala:121 Redis.deleteKeysByPattern(s"${RATE_LIMIT_ACTIVE_PREFIX}${id}_*") * * Invalidation is pattern matching over the whole key. If the derivation grows a prefix, changes @@ -129,6 +130,24 @@ class CacheKeyFormatTest extends FlatSpec with Matchers { InMemory.countKeys(glob) should be >= 1 } + it should "keep the pattern dynamic entity invalidation depends on matchable" in { + // NewStyle.invalidateDynamicEntityCache issues deleteKeysByPattern("*getDynamicEntities*") + // (through DynamicEntityHelper.forgetDefinitions), so keys derived from callers named + // getDynamicEntities and getDynamicEntitiesByUserId must both be matched by that glob. + val glob = "*getDynamicEntities*" + val regex = glob.replace("*", ".*") + Seq("getDynamicEntities", "getDynamicEntitiesByUserId").foreach { method => + val marker = s"(CacheKeyFormatTest,$method,${java.util.UUID.randomUUID().toString.take(8)})" + val before = storedKeys + Caching.memoizeSyncWithImMemory(Some(marker))(ttl)("definitions") + val added = (storedKeys -- before).head + withClue(s"derived key '$added' is not matched by the invalidation pattern '$glob'. " + + s"A changed dynamic entity definition would stay invisible for dynamicEntity.cache.ttl.seconds. ") { + added.matches(regex) shouldBe true + } + } + } + it should "keep the pattern rate-limit invalidation depends on matchable" in { // Caching.invalidateRateLimitCache(consumerId) issues // deleteKeysByPattern(s"*${RATE_LIMIT_ACTIVE_PREFIX}${consumerId}_*"). The cache key diff --git a/obp-api/src/test/scala/code/api/dynamic/entity/helper/DefinitionsCacheInvalidationTest.scala b/obp-api/src/test/scala/code/api/dynamic/entity/helper/DefinitionsCacheInvalidationTest.scala new file mode 100644 index 0000000000..1eaffc19f8 --- /dev/null +++ b/obp-api/src/test/scala/code/api/dynamic/entity/helper/DefinitionsCacheInvalidationTest.scala @@ -0,0 +1,101 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ + +package code.api.dynamic.entity.helper + +import code.api.cache.{Caching, Redis} +import code.api.util.http4s.RequestScopeConnection +import code.setup.RedisTestTarget +import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} + +import java.util.concurrent.ConcurrentLinkedQueue +import scala.concurrent.duration._ + +/** + * A changed dynamic entity definition must not be served from a list cached before the change. + * + * NewStyle.getDynamicEntities memoizes the definition list in Redis for dynamicEntity.cache.ttl.seconds + * (default 30), and DynamicEntityHelper.buildDefinitionsMap reads that list. When forgetDefinitions only + * dropped the definitions map, its rebuild read the memoized list from before the change and kept it for + * dynamicEntity.definitions_map.cache.ttl.seconds (default 300): a new entity's record endpoints answered + * 404 for up to five minutes. Both caches are off in test mode, so these tests write the memoized entry + * themselves, under a key carrying the method name the way the real one does (CacheKeyFormatTest). + */ +class DefinitionsCacheInvalidationTest extends FlatSpec with Matchers with BeforeAndAfterEach { + + private def memoizedListKey(): String = + s"(DefinitionsCacheInvalidationTest,getDynamicEntities,${java.util.UUID.randomUUID().toString.take(8)})" + + private def remember(key: String): Unit = + Caching.memoizeSyncWithProvider(Some(key))(30.seconds)("definitions as they were before the change") + + private def stored(key: String): Int = Redis.countKeys(s"*$key*") + + private def requireRedis(): Unit = + RedisTestTarget.requireReachable(Redis.isRedisReady, "the dynamic entity definition cache checks") + + override def afterEach(): Unit = { + RequestScopeConnection.currentAfterCommit.remove() + super.afterEach() + } + + "forgetDefinitions" should "drop the memoized definition lists, so the rebuild reads the current rows" in { + requireRedis() + val key = memoizedListKey() + remember(key) + stored(key) shouldBe 1 + + DynamicEntityHelper.forgetDefinitions() + + withClue("a list memoized before the change survived forgetDefinitions; the definitions map would be " + + "rebuilt from it and the change would stay invisible for the map's whole time-to-live ") { + stored(key) shouldBe 0 + } + } + + it should "drop them again once the transaction commits" in { + // Inside a request transaction the change is not visible to other connections until commit. A list read + // between forgetDefinitions and the commit holds the old rows and has to be dropped after the commit. + requireRedis() + val pending = new ConcurrentLinkedQueue[() => Unit]() + RequestScopeConnection.currentAfterCommit.set(pending) + + DynamicEntityHelper.forgetDefinitions() + val readBeforeCommit = memoizedListKey() + remember(readBeforeCommit) + stored(readBeforeCommit) shouldBe 1 + + withClue("forgetDefinitions registered nothing to run after the commit ") { + pending.isEmpty shouldBe false + } + Iterator.continually(pending.poll()).takeWhile(_ != null).foreach(_.apply()) + + withClue("a list read before the commit survived it ") { + stored(readBeforeCommit) shouldBe 0 + } + } +} From b877346763eb74e2e847e7a57c872fde218a571a Mon Sep 17 00:00:00 2001 From: Hongwei Date: Mon, 2 Nov 2026 17:54:05 +0100 Subject: [PATCH 2/2] fix(dynamic-entity): follow a dynamic_entity cache namespace version for definitions A definition change now bumps a new dynamic_entity cache namespace once its transaction has committed, instead of deleting the memoized lists by pattern. The lists NewStyle.getDynamicEntities and getDynamicEntitiesByUserId memoize carry that version in their keys, and DynamicEntityHelper keeps its definitions map (DefinitionsMapCache) only while the version is the one it was built under, so every instance reads the definitions again within a second, and a read still in flight cannot store old rows under the new version. The dynamic resource-doc caches are bumped in the same after-commit step rather than inside the request transaction. The namespace is listed on the cache info page and can be bumped through the cache namespace endpoint. --- .../scala/code/api/constant/constant.scala | 7 +- .../entity/helper/DefinitionsMapCache.scala | 68 ++++++++ .../entity/helper/DynamicEntityHelper.scala | 64 +++---- .../main/scala/code/api/util/NewStyle.scala | 67 +++++--- .../code/api/v6_0_0/JSONFactory6.0.0.scala | 6 +- .../code/api/cache/CacheKeyFormatTest.scala | 19 -- .../DefinitionsCacheInvalidationTest.scala | 162 ++++++++++++++---- 7 files changed, 274 insertions(+), 119 deletions(-) create mode 100644 obp-api/src/main/scala/code/api/dynamic/entity/helper/DefinitionsMapCache.scala diff --git a/obp-api/src/main/scala/code/api/constant/constant.scala b/obp-api/src/main/scala/code/api/constant/constant.scala index e684eb40d8..e32020497c 100644 --- a/obp-api/src/main/scala/code/api/constant/constant.scala +++ b/obp-api/src/main/scala/code/api/constant/constant.scala @@ -374,6 +374,10 @@ object Constant extends MdcLoggable { // The Glossary as each instance holds it in memory. Bumping it reloads the Glossary at once and // rebuilds every cached resource-docs document, because their keys carry the Glossary version. final val GLOSSARY_NAMESPACE = "glossary" + // Dynamic entity definitions: the definition lists memoized in Redis and the definitions map each + // instance builds from them. A definition change bumps it once committed, so every instance reads + // the definitions again. + final val DYNAMIC_ENTITY_NAMESPACE = "dynamic_entity" // List of all versioned cache namespaces final val ALL_CACHE_NAMESPACES = List( @@ -393,7 +397,8 @@ object Constant extends MdcLoggable { FINANCIAL_PRODUCTS_NAMESPACE, API_PRODUCTS_NAMESPACE, MESSAGE_DOCS_NAMESPACE, - GLOSSARY_NAMESPACE + GLOSSARY_NAMESPACE, + DYNAMIC_ENTITY_NAMESPACE ) // Cache key prefixes with global namespace and versioning for easy invalidation diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/helper/DefinitionsMapCache.scala b/obp-api/src/main/scala/code/api/dynamic/entity/helper/DefinitionsMapCache.scala new file mode 100644 index 0000000000..1cc717cb64 --- /dev/null +++ b/obp-api/src/main/scala/code/api/dynamic/entity/helper/DefinitionsMapCache.scala @@ -0,0 +1,68 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ + +package code.api.dynamic.entity.helper + +/** + * Keeps one built value (the dynamic entity definitions map) for up to `ttlMillis`, and only while + * `currentVersion` still returns the version it was built under. + * + * The version is read before building, so a value built while the version changed is kept under the + * old version and dropped on the next read. [[forget]] drops the value on this instance alone, for when + * the version cannot change (Redis unavailable); a build that started before it is not kept after it. + * Nothing is kept, and the version is not read, when `ttlMillis` is not positive. + */ +private[helper] final class DefinitionsMapCache[M](ttlMillis: Long, currentVersion: () => Long, build: () => M) { + + // (built at, version it was built under, value) + @volatile private var entry: Option[(Long, Long, M)] = None + // Replaced by forget; a build keeps its value only if the token it started under is still current. + @volatile private var token: AnyRef = new Object + + def get: M = + if (ttlMillis <= 0) build() + else { + val now = System.currentTimeMillis() + val version = currentVersion() + entry match { + case Some((builtAt, builtUnder, value)) if builtUnder == version && now - builtAt < ttlMillis => value + case _ => + val startedUnder = token + val value = build() + if (token eq startedUnder) entry = Some((now, version, value)) + value + } + } + + def forget(): Unit = { + token = new Object + entry = None + } + + /** The value kept now, if any, whatever its version. */ + def kept: Option[M] = entry.map(_._3) +} diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala b/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala index 03169dfbdc..5aea0982a8 100644 --- a/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala +++ b/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala @@ -27,6 +27,7 @@ TESOBE (http://www.tesobe.com/) package code.api.dynamic.entity.helper +import code.api.Constant import code.api.util.APIUtil.{EmptyBody, ResourceDoc, userAuthenticationMessage} import code.api.util.ApiRole.getOrCreateDynamicApiRole import code.api.util.ApiTag._ @@ -202,53 +203,44 @@ object DynamicEntityHelper { // It is asked for many times per request (the access checks, each field's read check, the resource // docs), and building it reads every definition and parses each one's JSON: with 120 definitions // that was ~5 ms a time, and a Dynamic Query returning 11 records spent ~0.5 s rebuilding it. So - // the built map is kept for dynamicEntity.definitions_map.cache.ttl.seconds (default 300), and - // forgotten the moment a definition is created, updated or deleted on this node - // (MappedDynamicEntityProvider calls forgetDefinitions). Another node sees such a change when its - // copy expires. Not kept in test mode, where a test may change the table behind the provider. + // the built map is kept for dynamicEntity.definitions_map.cache.ttl.seconds (default 300), and only + // while the dynamic_entity cache namespace version is the one it was built under: a definition change + // bumps that version once committed (forgetDefinitions), and every instance, which re-reads the version + // at most once a second, then builds the map again. Not kept in test mode, where a test may change the + // table behind the provider. private val definitionsMapTtlMillis: Long = if (net.liftweb.util.Props.testMode) 0L else APIUtil.getPropsAsLongValue("dynamicEntity.definitions_map.cache.ttl.seconds", 300L) * 1000L - // (built at, map). Each change replaces the token, and a build keeps its map only while the token it - // started under is still the current one, so a build that started before a change is not kept after it. - @volatile private var definitionsMapCache: Option[(Long, Map[(String, String), DynamicEntityInfo])] = None - @volatile private var definitionsToken: AnyRef = new Object + + private val definitionsMapCache = new DefinitionsMapCache[Map[(String, String), DynamicEntityInfo]]( + definitionsMapTtlMillis, + () => Constant.recentCacheNamespaceVersion(Constant.DYNAMIC_ENTITY_NAMESPACE), + () => buildDefinitionsMap) /** - * Forget the kept definitions map and the memoized definition lists, so the next use reads the definitions - * again. The map is rebuilt from NewStyle.getDynamicEntities, which keeps its list in Redis for - * dynamicEntity.cache.ttl.seconds: forgetting only the map rebuilt it from the list as it was before the - * change, and kept that for the map's whole time-to-live. + * Tell every instance that a definition was created, updated or deleted, once the request's transaction + * has committed (at once outside a transaction): bump the dynamic_entity cache namespace version, which + * the definitions map and the definition lists NewStyle.getDynamicEntities memoizes in Redis both carry, + * so all of them read the definitions again, and then the dynamic resource-doc caches, which hold the + * docs generated from the definitions. Not earlier, because until the commit the change is not visible + * to other connections: an instance that saw a bump first could read the old rows and keep them under the + * new version. Nothing happens if the transaction rolls back. * - * This node's map is dropped at once, for reads later in the same request. The map and the lists are dropped - * again once the request's transaction commits (at once outside a transaction): until then the change is not - * visible to other connections, so anything read in between holds the old definitions, and clearing the - * shared lists earlier would achieve nothing. Other nodes' maps still follow their own time-to-live. + * This instance also drops its map directly, which is what keeps it current when Redis is unavailable + * and the version cannot change. Another instance follows the dynamic_entity version within a second, so + * for that second it can still build resource docs from its previous map. */ - def forgetDefinitions(): Unit = { - dropDefinitionsMap() + def forgetDefinitions(): Unit = code.api.util.http4s.RequestScopeConnection.afterCommit { - dropDefinitionsMap() - NewStyle.function.invalidateDynamicEntityCache() + Constant.incrementCacheNamespaceVersion(Constant.DYNAMIC_ENTITY_NAMESPACE) + definitionsMapCache.forget() + NewStyle.function.invalidateDynamicResourceDocCaches() } - } - private def dropDefinitionsMap(): Unit = { - definitionsToken = new Object - definitionsMapCache = None - } + def definitionsMap: Map[(String, String), DynamicEntityInfo] = definitionsMapCache.get - def definitionsMap: Map[(String, String), DynamicEntityInfo] = { - val now = System.currentTimeMillis() - definitionsMapCache match { - case Some((builtAt, map)) if now - builtAt < definitionsMapTtlMillis => map - case _ => - val token = definitionsToken - val map = buildDefinitionsMap - if (definitionsMapTtlMillis > 0 && (definitionsToken eq token)) definitionsMapCache = Some((now, map)) - map - } - } + /** The number of definitions in this instance's kept map, for the cache info page. */ + def keptDefinitionCount: Int = definitionsMapCache.kept.map(_.size).getOrElse(0) private def buildDefinitionsMap: Map[(String, String), DynamicEntityInfo] = NewStyle.function.getDynamicEntities(None, true).map(it => ((DynamicEntitySpace.bankIdOrSystem(it.bankId), it.entityName), DynamicEntityInfo(it.metadataJson, it.entityName, it.bankId, it.hasPersonalEntity, it.hasPublicAccess, it.hasCommunityAccess, it.personalRequiresRole, it.useRowLevelAccess, it.authMode))).toMap diff --git a/obp-api/src/main/scala/code/api/util/NewStyle.scala b/obp-api/src/main/scala/code/api/util/NewStyle.scala index 6c7e2644b4..663edf15e6 100644 --- a/obp-api/src/main/scala/code/api/util/NewStyle.scala +++ b/obp-api/src/main/scala/code/api/util/NewStyle.scala @@ -3543,12 +3543,14 @@ object NewStyle extends MdcLoggable{ * next `/resource-docs` request regenerates from the database instead of * serving a pre-change snapshot (TTL is 1 hour by default). * - * Call after a Dynamic Entity, Dynamic Endpoint or Dynamic Resource Doc is - * created, updated, or deleted: each one generates resource docs. The static - * resource-doc / swagger caches are not touched because dynamic docs only - * reach them through the content=all documents, which are left to their TTL. + * Call after a Dynamic Endpoint or Dynamic Resource Doc is created, updated, + * or deleted: each one generates resource docs. For a Dynamic Entity, + * DynamicEntityHelper.forgetDefinitions calls it once the change has + * committed. The static resource-doc / swagger caches are not touched because + * dynamic docs only reach them through the content=all documents, which are + * left to their TTL. */ - private def invalidateDynamicResourceDocCaches(): Unit = { + private[code] def invalidateDynamicResourceDocCaches(): Unit = { Constant.incrementCacheNamespaceVersion(Constant.RD_DYNAMIC_NAMESPACE) Constant.incrementCacheNamespaceVersion(Constant.RD_ALL_NAMESPACE) } @@ -3589,9 +3591,9 @@ object NewStyle extends MdcLoggable{ } Future { - val result = DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate(dynamicEntity) - if (result.isDefined) invalidateDynamicResourceDocCaches() - result + // The provider calls DynamicEntityHelper.forgetDefinitions, which also drops the dynamic + // resource-doc caches once the change has committed. + DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate(dynamicEntity) } } @@ -3630,9 +3632,9 @@ object NewStyle extends MdcLoggable{ } Future { - val result = DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate(dynamicEntity) - if (result.isDefined) invalidateDynamicResourceDocCaches() - result + // The provider calls DynamicEntityHelper.forgetDefinitions, which also drops the dynamic + // resource-doc caches once the change has committed. + DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate(dynamicEntity) } } @@ -3674,8 +3676,8 @@ object NewStyle extends MdcLoggable{ } deleteEntitleMentResult } - // The entity (and its generated resource docs) are gone — drop the stale doc caches. - if (result.exists(deleted => deleted)) invalidateDynamicResourceDocCaches() + // The entity's generated resource docs go with it: the provider's delete calls + // DynamicEntityHelper.forgetDefinitions, which drops the doc caches once the delete has committed. result } } @@ -3704,9 +3706,27 @@ object NewStyle extends MdcLoggable{ } def getDynamicEntities(bankId: Option[String], returnBothBankAndSystemLevel: Boolean): List[DynamicEntityT] = { - import scala.concurrent.duration._ - validateBankId(bankId, None) + getDynamicEntities(bankId, returnBothBankAndSystemLevel, dynamicEntityDefinitionsVersion) + } + + /** + * The dynamic_entity cache namespace version, which the memoized definition lists below carry in their + * keys. DynamicEntityHelper.forgetDefinitions bumps it once a definition change has committed, so every + * list memoized before the change becomes unreachable on every instance, including one that a read still + * in flight writes after the bump: it is written under the version that read started with. + */ + private def dynamicEntityDefinitionsVersion: Long = + if (dynamicEntityTTL == 0) 0L // nothing is memoized, so no need to ask Redis + else Constant.recentCacheNamespaceVersion(Constant.DYNAMIC_ENTITY_NAMESPACE) + + // The version is a parameter so that CacheKeyFromArguments renders it into the key; the name stays + // getDynamicEntities so the key and its Telemetry label stay the same apart from the version. Keep + // buildCacheKey as the tail expression (see getEndpointMappingsCached): bound to a val first, the macro + // renders no arguments and every list would share one key whatever the version. Test mode memoizes + // nothing, so no test would notice; the rendering was checked with javap. + private def getDynamicEntities(bankId: Option[String], returnBothBankAndSystemLevel: Boolean, definitionsVersion: Long): List[DynamicEntityT] = { + import scala.concurrent.duration._ var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString) CacheKeyFromArguments.buildCacheKey { @@ -3715,20 +3735,11 @@ object NewStyle extends MdcLoggable{ } } } - - /** - * Drop every memoized `getDynamicEntities(...)` and `getDynamicEntitiesByUserId(...)` list after a - * definition is created, updated or deleted, so the change shows in the next read instead of after - * `dynamicEntity.cache.ttl.seconds`. As with `invalidateMethodRoutingCache`, the Redis memoize key embeds - * the method name, so one pattern delete clears every argument variant, for every node reading Redis. - * Called through DynamicEntityHelper.forgetDefinitions once the change has committed. No-op / logged when - * Redis is unavailable. - */ - def invalidateDynamicEntityCache(): Unit = { - Redis.deleteKeysByPattern("*getDynamicEntities*") - } - def getDynamicEntitiesByUserId(userId: String): List[DynamicEntityT] = { + def getDynamicEntitiesByUserId(userId: String): List[DynamicEntityT] = + getDynamicEntitiesByUserId(userId, dynamicEntityDefinitionsVersion) + + private def getDynamicEntitiesByUserId(userId: String, definitionsVersion: Long): List[DynamicEntityT] = { import scala.concurrent.duration._ var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString) diff --git a/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala b/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala index 99f0cc9d8a..21a90e8fd4 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala @@ -2444,14 +2444,16 @@ object JSONFactory600 extends CustomJsonFormats with MdcLoggable { Constant.FINANCIAL_PRODUCTS_NAMESPACE -> ("Financial product list (bank-scoped and all-banks)", "Products"), Constant.API_PRODUCTS_NAMESPACE -> ("Api product list (all banks)", "Products"), Constant.MESSAGE_DOCS_NAMESPACE -> ("Message docs and connector JSON Schemas (Redis, and each instance's memory)", "API Documentation"), - Constant.GLOSSARY_NAMESPACE -> ("Glossary items held in each instance's memory; bumping also rebuilds cached resource docs", "API Documentation") + Constant.GLOSSARY_NAMESPACE -> ("Glossary items held in each instance's memory; bumping also rebuilds cached resource docs", "API Documentation"), + Constant.DYNAMIC_ENTITY_NAMESPACE -> ("Dynamic entity definitions: the definition lists in Redis and each instance's definitions map", "Dynamic Entities") ) // Caches held in this instance's memory outside the shared in-memory store, which follow their // namespace's version through their own keys. Counted here so the page shows their real size. val inProcessEntries: Map[String, () => Long] = Map( Constant.MESSAGE_DOCS_NAMESPACE -> (() => code.api.v2_2_0.MessageDocsJsonCache.size + code.api.util.JsonSchemaGenerator.cacheSize), - Constant.GLOSSARY_NAMESPACE -> (() => code.api.util.Glossary.loadedItemCount.toLong) + Constant.GLOSSARY_NAMESPACE -> (() => code.api.util.Glossary.loadedItemCount.toLong), + Constant.DYNAMIC_ENTITY_NAMESPACE -> (() => code.api.dynamic.entity.helper.DynamicEntityHelper.keptDefinitionCount.toLong) ) var redisAvailable = true diff --git a/obp-api/src/test/scala/code/api/cache/CacheKeyFormatTest.scala b/obp-api/src/test/scala/code/api/cache/CacheKeyFormatTest.scala index 0fc3972f98..0a97c8326b 100644 --- a/obp-api/src/test/scala/code/api/cache/CacheKeyFormatTest.scala +++ b/obp-api/src/test/scala/code/api/cache/CacheKeyFormatTest.scala @@ -48,7 +48,6 @@ import scala.jdk.CollectionConverters._ * is in there somewhere. That is not enough for the thing that actually depends on this format: * * NewStyle.scala:3306 Redis.deleteKeysByPattern("*getMethodRoutings*") - * NewStyle.scala Redis.deleteKeysByPattern("*getDynamicEntities*") * Caching.scala:121 Redis.deleteKeysByPattern(s"${RATE_LIMIT_ACTIVE_PREFIX}${id}_*") * * Invalidation is pattern matching over the whole key. If the derivation grows a prefix, changes @@ -130,24 +129,6 @@ class CacheKeyFormatTest extends FlatSpec with Matchers { InMemory.countKeys(glob) should be >= 1 } - it should "keep the pattern dynamic entity invalidation depends on matchable" in { - // NewStyle.invalidateDynamicEntityCache issues deleteKeysByPattern("*getDynamicEntities*") - // (through DynamicEntityHelper.forgetDefinitions), so keys derived from callers named - // getDynamicEntities and getDynamicEntitiesByUserId must both be matched by that glob. - val glob = "*getDynamicEntities*" - val regex = glob.replace("*", ".*") - Seq("getDynamicEntities", "getDynamicEntitiesByUserId").foreach { method => - val marker = s"(CacheKeyFormatTest,$method,${java.util.UUID.randomUUID().toString.take(8)})" - val before = storedKeys - Caching.memoizeSyncWithImMemory(Some(marker))(ttl)("definitions") - val added = (storedKeys -- before).head - withClue(s"derived key '$added' is not matched by the invalidation pattern '$glob'. " + - s"A changed dynamic entity definition would stay invisible for dynamicEntity.cache.ttl.seconds. ") { - added.matches(regex) shouldBe true - } - } - } - it should "keep the pattern rate-limit invalidation depends on matchable" in { // Caching.invalidateRateLimitCache(consumerId) issues // deleteKeysByPattern(s"*${RATE_LIMIT_ACTIVE_PREFIX}${consumerId}_*"). The cache key diff --git a/obp-api/src/test/scala/code/api/dynamic/entity/helper/DefinitionsCacheInvalidationTest.scala b/obp-api/src/test/scala/code/api/dynamic/entity/helper/DefinitionsCacheInvalidationTest.scala index 1eaffc19f8..516ee8e3e2 100644 --- a/obp-api/src/test/scala/code/api/dynamic/entity/helper/DefinitionsCacheInvalidationTest.scala +++ b/obp-api/src/test/scala/code/api/dynamic/entity/helper/DefinitionsCacheInvalidationTest.scala @@ -27,75 +27,171 @@ TESOBE (http://www.tesobe.com/) package code.api.dynamic.entity.helper -import code.api.cache.{Caching, Redis} +import code.api.Constant +import code.api.cache.Redis import code.api.util.http4s.RequestScopeConnection import code.setup.RedisTestTarget import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} import java.util.concurrent.ConcurrentLinkedQueue -import scala.concurrent.duration._ /** - * A changed dynamic entity definition must not be served from a list cached before the change. + * A changed dynamic entity definition must reach every instance's definition caches. * - * NewStyle.getDynamicEntities memoizes the definition list in Redis for dynamicEntity.cache.ttl.seconds - * (default 30), and DynamicEntityHelper.buildDefinitionsMap reads that list. When forgetDefinitions only - * dropped the definitions map, its rebuild read the memoized list from before the change and kept it for - * dynamicEntity.definitions_map.cache.ttl.seconds (default 300): a new entity's record endpoints answered - * 404 for up to five minutes. Both caches are off in test mode, so these tests write the memoized entry - * themselves, under a key carrying the method name the way the real one does (CacheKeyFormatTest). + * Two caches hold definitions: the definition lists NewStyle.getDynamicEntities memoizes in Redis + * (dynamicEntity.cache.ttl.seconds, default 30) and the definitions map each instance builds from them + * (dynamicEntity.definitions_map.cache.ttl.seconds, default 300). Both follow the dynamic_entity cache + * namespace version, which a definition change bumps once its transaction has committed. When a change + * only dropped this instance's map, the rebuild read the list memoized before the change and kept it for + * the map's whole time-to-live, and other instances never heard of the change at all: a new entity's + * record endpoints answered 404 for up to five minutes. */ class DefinitionsCacheInvalidationTest extends FlatSpec with Matchers with BeforeAndAfterEach { - private def memoizedListKey(): String = - s"(DefinitionsCacheInvalidationTest,getDynamicEntities,${java.util.UUID.randomUUID().toString.take(8)})" + private val namespace = "dynamic_entity" - private def remember(key: String): Unit = - Caching.memoizeSyncWithProvider(Some(key))(30.seconds)("definitions as they were before the change") - - private def stored(key: String): Int = Redis.countKeys(s"*$key*") + private def version(): Long = Constant.getCacheNamespaceVersion(namespace) private def requireRedis(): Unit = RedisTestTarget.requireReachable(Redis.isRedisReady, "the dynamic entity definition cache checks") + private def runPending(pending: ConcurrentLinkedQueue[() => Unit]): Unit = + Iterator.continually(pending.poll()).takeWhile(_ != null).foreach(_.apply()) + override def afterEach(): Unit = { RequestScopeConnection.currentAfterCommit.remove() super.afterEach() } - "forgetDefinitions" should "drop the memoized definition lists, so the rebuild reads the current rows" in { + "The dynamic_entity namespace" should "be listed with the other cache namespaces" in { + // Listed, it shows on the cache info page and an administrator can bump it by hand. + Constant.ALL_CACHE_NAMESPACES should contain (namespace) + } + + "forgetDefinitions" should "bump the dynamic_entity version once the transaction commits" in { + // Until the commit the change is not visible to other connections: an instance that saw the bump + // earlier could read the old rows and cache them under the new version. requireRedis() - val key = memoizedListKey() - remember(key) - stored(key) shouldBe 1 + val pending = new ConcurrentLinkedQueue[() => Unit]() + RequestScopeConnection.currentAfterCommit.set(pending) + val before = version() DynamicEntityHelper.forgetDefinitions() - withClue("a list memoized before the change survived forgetDefinitions; the definitions map would be " + - "rebuilt from it and the change would stay invisible for the map's whole time-to-live ") { - stored(key) shouldBe 0 + withClue("the version was bumped before the commit ") { + version() shouldBe before + } + runPending(pending) + withClue("the commit did not bump the dynamic_entity version, so no instance learns of the change ") { + version() shouldBe before + 1 } } - it should "drop them again once the transaction commits" in { - // Inside a request transaction the change is not visible to other connections until commit. A list read - // between forgetDefinitions and the commit holds the old rows and has to be dropped after the commit. + it should "bump the dynamic resource-doc caches only once the transaction commits" in { + // Dynamic entities generate resource docs. Bumped before the commit, another instance could rebuild + // /resource-docs from the old definitions and keep that for the resource-doc time-to-live (an hour). requireRedis() val pending = new ConcurrentLinkedQueue[() => Unit]() RequestScopeConnection.currentAfterCommit.set(pending) + val dynamicBefore = Constant.getCacheNamespaceVersion(Constant.RD_DYNAMIC_NAMESPACE) + val allBefore = Constant.getCacheNamespaceVersion(Constant.RD_ALL_NAMESPACE) DynamicEntityHelper.forgetDefinitions() - val readBeforeCommit = memoizedListKey() - remember(readBeforeCommit) - stored(readBeforeCommit) shouldBe 1 - withClue("forgetDefinitions registered nothing to run after the commit ") { - pending.isEmpty shouldBe false + withClue("the resource-doc caches were bumped before the commit ") { + Constant.getCacheNamespaceVersion(Constant.RD_DYNAMIC_NAMESPACE) shouldBe dynamicBefore + Constant.getCacheNamespaceVersion(Constant.RD_ALL_NAMESPACE) shouldBe allBefore } - Iterator.continually(pending.poll()).takeWhile(_ != null).foreach(_.apply()) + runPending(pending) + withClue("the commit did not bump the resource-doc caches, so /resource-docs keeps the old entities ") { + Constant.getCacheNamespaceVersion(Constant.RD_DYNAMIC_NAMESPACE) shouldBe dynamicBefore + 1 + Constant.getCacheNamespaceVersion(Constant.RD_ALL_NAMESPACE) shouldBe allBefore + 1 + } + } + + it should "bump it at once outside a transaction" in { + requireRedis() + val before = version() + DynamicEntityHelper.forgetDefinitions() + version() shouldBe before + 1 + } + + it should "leave it alone when the transaction rolls back" in { + requireRedis() + val pending = new ConcurrentLinkedQueue[() => Unit]() + RequestScopeConnection.currentAfterCommit.set(pending) + val before = version() - withClue("a list read before the commit survived it ") { - stored(readBeforeCommit) shouldBe 0 + DynamicEntityHelper.forgetDefinitions() + pending.clear() // what a rollback does with the after-commit actions + + version() shouldBe before + } + + "The definitions map cache" should "keep a built map while the version is unchanged" in { + val probe = new Probe + probe.cache.get shouldBe 1 + probe.cache.get shouldBe 1 + probe.builds shouldBe 1 + } + + it should "build it again once the version changes, as when another instance bumps it" in { + val probe = new Probe + probe.cache.get + probe.version += 1 + withClue("a map built under the previous version was served after the bump ") { + probe.cache.get shouldBe 2 } } + + it should "not keep a map built while the version changed under it" in { + // The build read the definitions before the change committed; it is kept under the version read + // before the build, which is already behind, so the next read builds again. + val probe = new Probe + probe.duringBuild = () => probe.version += 1 + probe.cache.get shouldBe 1 + probe.duringBuild = () => () + probe.cache.get shouldBe 2 + } + + it should "drop the map on forget even though the version did not change (Redis unavailable)" in { + val probe = new Probe + probe.cache.get + probe.cache.forget() + probe.cache.get shouldBe 2 + } + + it should "not keep a map whose build started before forget" in { + val probe = new Probe + probe.duringBuild = () => probe.cache.forget() + probe.cache.get shouldBe 1 + probe.duringBuild = () => () + probe.cache.get shouldBe 2 + } + + it should "keep nothing, and not read the version, when its time-to-live is zero" in { + val probe = new Probe(ttlMillis = 0L) + probe.cache.get shouldBe 1 + probe.cache.get shouldBe 2 + probe.versionReads shouldBe 0 + } + + it should "build it again once its time-to-live has passed" in { + val probe = new Probe(ttlMillis = 50L) + probe.cache.get + Thread.sleep(80) + probe.cache.get shouldBe 2 + } + + /** A map cache whose version and builds the test controls; each build returns its own number. */ + private class Probe(ttlMillis: Long = 60000L) { + @volatile var version: Long = 1L + @volatile var builds: Int = 0 + @volatile var versionReads: Int = 0 + @volatile var duringBuild: () => Unit = () => () + val cache = new DefinitionsMapCache[Int]( + ttlMillis, + () => { versionReads += 1; version }, + () => { builds += 1; duringBuild(); builds }) + } }