diff --git a/obp-api/src/main/scala/code/api/constant/constant.scala b/obp-api/src/main/scala/code/api/constant/constant.scala index e684eb40d8..e32020497c 100644 --- a/obp-api/src/main/scala/code/api/constant/constant.scala +++ b/obp-api/src/main/scala/code/api/constant/constant.scala @@ -374,6 +374,10 @@ object Constant extends MdcLoggable { // The Glossary as each instance holds it in memory. Bumping it reloads the Glossary at once and // rebuilds every cached resource-docs document, because their keys carry the Glossary version. final val GLOSSARY_NAMESPACE = "glossary" + // Dynamic entity definitions: the definition lists memoized in Redis and the definitions map each + // instance builds from them. A definition change bumps it once committed, so every instance reads + // the definitions again. + final val DYNAMIC_ENTITY_NAMESPACE = "dynamic_entity" // List of all versioned cache namespaces final val ALL_CACHE_NAMESPACES = List( @@ -393,7 +397,8 @@ object Constant extends MdcLoggable { FINANCIAL_PRODUCTS_NAMESPACE, API_PRODUCTS_NAMESPACE, MESSAGE_DOCS_NAMESPACE, - GLOSSARY_NAMESPACE + GLOSSARY_NAMESPACE, + DYNAMIC_ENTITY_NAMESPACE ) // Cache key prefixes with global namespace and versioning for easy invalidation diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/helper/DefinitionsMapCache.scala b/obp-api/src/main/scala/code/api/dynamic/entity/helper/DefinitionsMapCache.scala new file mode 100644 index 0000000000..1cc717cb64 --- /dev/null +++ b/obp-api/src/main/scala/code/api/dynamic/entity/helper/DefinitionsMapCache.scala @@ -0,0 +1,68 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ + +package code.api.dynamic.entity.helper + +/** + * Keeps one built value (the dynamic entity definitions map) for up to `ttlMillis`, and only while + * `currentVersion` still returns the version it was built under. + * + * The version is read before building, so a value built while the version changed is kept under the + * old version and dropped on the next read. [[forget]] drops the value on this instance alone, for when + * the version cannot change (Redis unavailable); a build that started before it is not kept after it. + * Nothing is kept, and the version is not read, when `ttlMillis` is not positive. + */ +private[helper] final class DefinitionsMapCache[M](ttlMillis: Long, currentVersion: () => Long, build: () => M) { + + // (built at, version it was built under, value) + @volatile private var entry: Option[(Long, Long, M)] = None + // Replaced by forget; a build keeps its value only if the token it started under is still current. + @volatile private var token: AnyRef = new Object + + def get: M = + if (ttlMillis <= 0) build() + else { + val now = System.currentTimeMillis() + val version = currentVersion() + entry match { + case Some((builtAt, builtUnder, value)) if builtUnder == version && now - builtAt < ttlMillis => value + case _ => + val startedUnder = token + val value = build() + if (token eq startedUnder) entry = Some((now, version, value)) + value + } + } + + def forget(): Unit = { + token = new Object + entry = None + } + + /** The value kept now, if any, whatever its version. */ + def kept: Option[M] = entry.map(_._3) +} diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala b/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala index 9043c19a11..5aea0982a8 100644 --- a/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala +++ b/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala @@ -27,6 +27,7 @@ TESOBE (http://www.tesobe.com/) package code.api.dynamic.entity.helper +import code.api.Constant import code.api.util.APIUtil.{EmptyBody, ResourceDoc, userAuthenticationMessage} import code.api.util.ApiRole.getOrCreateDynamicApiRole import code.api.util.ApiTag._ @@ -202,35 +203,44 @@ object DynamicEntityHelper { // It is asked for many times per request (the access checks, each field's read check, the resource // docs), and building it reads every definition and parses each one's JSON: with 120 definitions // that was ~5 ms a time, and a Dynamic Query returning 11 records spent ~0.5 s rebuilding it. So - // the built map is kept for dynamicEntity.definitions_map.cache.ttl.seconds (default 300), and - // forgotten the moment a definition is created, updated or deleted on this node - // (MappedDynamicEntityProvider calls forgetDefinitions). Another node sees such a change when its - // copy expires. Not kept in test mode, where a test may change the table behind the provider. + // the built map is kept for dynamicEntity.definitions_map.cache.ttl.seconds (default 300), and only + // while the dynamic_entity cache namespace version is the one it was built under: a definition change + // bumps that version once committed (forgetDefinitions), and every instance, which re-reads the version + // at most once a second, then builds the map again. Not kept in test mode, where a test may change the + // table behind the provider. private val definitionsMapTtlMillis: Long = if (net.liftweb.util.Props.testMode) 0L else APIUtil.getPropsAsLongValue("dynamicEntity.definitions_map.cache.ttl.seconds", 300L) * 1000L - // (built at, map). Each change replaces the token, and a build keeps its map only while the token it - // started under is still the current one, so a build that started before a change is not kept after it. - @volatile private var definitionsMapCache: Option[(Long, Map[(String, String), DynamicEntityInfo])] = None - @volatile private var definitionsToken: AnyRef = new Object - - /** Forget the kept definitions map, so the next use reads the definitions again. */ - def forgetDefinitions(): Unit = { - definitionsToken = new Object - definitionsMapCache = None - } - def definitionsMap: Map[(String, String), DynamicEntityInfo] = { - val now = System.currentTimeMillis() - definitionsMapCache match { - case Some((builtAt, map)) if now - builtAt < definitionsMapTtlMillis => map - case _ => - val token = definitionsToken - val map = buildDefinitionsMap - if (definitionsMapTtlMillis > 0 && (definitionsToken eq token)) definitionsMapCache = Some((now, map)) - map + private val definitionsMapCache = new DefinitionsMapCache[Map[(String, String), DynamicEntityInfo]]( + definitionsMapTtlMillis, + () => Constant.recentCacheNamespaceVersion(Constant.DYNAMIC_ENTITY_NAMESPACE), + () => buildDefinitionsMap) + + /** + * Tell every instance that a definition was created, updated or deleted, once the request's transaction + * has committed (at once outside a transaction): bump the dynamic_entity cache namespace version, which + * the definitions map and the definition lists NewStyle.getDynamicEntities memoizes in Redis both carry, + * so all of them read the definitions again, and then the dynamic resource-doc caches, which hold the + * docs generated from the definitions. Not earlier, because until the commit the change is not visible + * to other connections: an instance that saw a bump first could read the old rows and keep them under the + * new version. Nothing happens if the transaction rolls back. + * + * This instance also drops its map directly, which is what keeps it current when Redis is unavailable + * and the version cannot change. Another instance follows the dynamic_entity version within a second, so + * for that second it can still build resource docs from its previous map. + */ + def forgetDefinitions(): Unit = + code.api.util.http4s.RequestScopeConnection.afterCommit { + Constant.incrementCacheNamespaceVersion(Constant.DYNAMIC_ENTITY_NAMESPACE) + definitionsMapCache.forget() + NewStyle.function.invalidateDynamicResourceDocCaches() } - } + + def definitionsMap: Map[(String, String), DynamicEntityInfo] = definitionsMapCache.get + + /** The number of definitions in this instance's kept map, for the cache info page. */ + def keptDefinitionCount: Int = definitionsMapCache.kept.map(_.size).getOrElse(0) private def buildDefinitionsMap: Map[(String, String), DynamicEntityInfo] = NewStyle.function.getDynamicEntities(None, true).map(it => ((DynamicEntitySpace.bankIdOrSystem(it.bankId), it.entityName), DynamicEntityInfo(it.metadataJson, it.entityName, it.bankId, it.hasPersonalEntity, it.hasPublicAccess, it.hasCommunityAccess, it.personalRequiresRole, it.useRowLevelAccess, it.authMode))).toMap diff --git a/obp-api/src/main/scala/code/api/util/NewStyle.scala b/obp-api/src/main/scala/code/api/util/NewStyle.scala index e2194dbdf8..663edf15e6 100644 --- a/obp-api/src/main/scala/code/api/util/NewStyle.scala +++ b/obp-api/src/main/scala/code/api/util/NewStyle.scala @@ -3543,12 +3543,14 @@ object NewStyle extends MdcLoggable{ * next `/resource-docs` request regenerates from the database instead of * serving a pre-change snapshot (TTL is 1 hour by default). * - * Call after a Dynamic Entity, Dynamic Endpoint or Dynamic Resource Doc is - * created, updated, or deleted: each one generates resource docs. The static - * resource-doc / swagger caches are not touched because dynamic docs only - * reach them through the content=all documents, which are left to their TTL. + * Call after a Dynamic Endpoint or Dynamic Resource Doc is created, updated, + * or deleted: each one generates resource docs. For a Dynamic Entity, + * DynamicEntityHelper.forgetDefinitions calls it once the change has + * committed. The static resource-doc / swagger caches are not touched because + * dynamic docs only reach them through the content=all documents, which are + * left to their TTL. */ - private def invalidateDynamicResourceDocCaches(): Unit = { + private[code] def invalidateDynamicResourceDocCaches(): Unit = { Constant.incrementCacheNamespaceVersion(Constant.RD_DYNAMIC_NAMESPACE) Constant.incrementCacheNamespaceVersion(Constant.RD_ALL_NAMESPACE) } @@ -3589,9 +3591,9 @@ object NewStyle extends MdcLoggable{ } Future { - val result = DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate(dynamicEntity) - if (result.isDefined) invalidateDynamicResourceDocCaches() - result + // The provider calls DynamicEntityHelper.forgetDefinitions, which also drops the dynamic + // resource-doc caches once the change has committed. + DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate(dynamicEntity) } } @@ -3630,9 +3632,9 @@ object NewStyle extends MdcLoggable{ } Future { - val result = DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate(dynamicEntity) - if (result.isDefined) invalidateDynamicResourceDocCaches() - result + // The provider calls DynamicEntityHelper.forgetDefinitions, which also drops the dynamic + // resource-doc caches once the change has committed. + DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate(dynamicEntity) } } @@ -3674,8 +3676,8 @@ object NewStyle extends MdcLoggable{ } deleteEntitleMentResult } - // The entity (and its generated resource docs) are gone — drop the stale doc caches. - if (result.exists(deleted => deleted)) invalidateDynamicResourceDocCaches() + // The entity's generated resource docs go with it: the provider's delete calls + // DynamicEntityHelper.forgetDefinitions, which drops the doc caches once the delete has committed. result } } @@ -3704,9 +3706,27 @@ object NewStyle extends MdcLoggable{ } def getDynamicEntities(bankId: Option[String], returnBothBankAndSystemLevel: Boolean): List[DynamicEntityT] = { - import scala.concurrent.duration._ - validateBankId(bankId, None) + getDynamicEntities(bankId, returnBothBankAndSystemLevel, dynamicEntityDefinitionsVersion) + } + + /** + * The dynamic_entity cache namespace version, which the memoized definition lists below carry in their + * keys. DynamicEntityHelper.forgetDefinitions bumps it once a definition change has committed, so every + * list memoized before the change becomes unreachable on every instance, including one that a read still + * in flight writes after the bump: it is written under the version that read started with. + */ + private def dynamicEntityDefinitionsVersion: Long = + if (dynamicEntityTTL == 0) 0L // nothing is memoized, so no need to ask Redis + else Constant.recentCacheNamespaceVersion(Constant.DYNAMIC_ENTITY_NAMESPACE) + + // The version is a parameter so that CacheKeyFromArguments renders it into the key; the name stays + // getDynamicEntities so the key and its Telemetry label stay the same apart from the version. Keep + // buildCacheKey as the tail expression (see getEndpointMappingsCached): bound to a val first, the macro + // renders no arguments and every list would share one key whatever the version. Test mode memoizes + // nothing, so no test would notice; the rendering was checked with javap. + private def getDynamicEntities(bankId: Option[String], returnBothBankAndSystemLevel: Boolean, definitionsVersion: Long): List[DynamicEntityT] = { + import scala.concurrent.duration._ var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString) CacheKeyFromArguments.buildCacheKey { @@ -3715,8 +3735,11 @@ object NewStyle extends MdcLoggable{ } } } - - def getDynamicEntitiesByUserId(userId: String): List[DynamicEntityT] = { + + def getDynamicEntitiesByUserId(userId: String): List[DynamicEntityT] = + getDynamicEntitiesByUserId(userId, dynamicEntityDefinitionsVersion) + + private def getDynamicEntitiesByUserId(userId: String, definitionsVersion: Long): List[DynamicEntityT] = { import scala.concurrent.duration._ var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString) diff --git a/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala b/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala index 99f0cc9d8a..21a90e8fd4 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala @@ -2444,14 +2444,16 @@ object JSONFactory600 extends CustomJsonFormats with MdcLoggable { Constant.FINANCIAL_PRODUCTS_NAMESPACE -> ("Financial product list (bank-scoped and all-banks)", "Products"), Constant.API_PRODUCTS_NAMESPACE -> ("Api product list (all banks)", "Products"), Constant.MESSAGE_DOCS_NAMESPACE -> ("Message docs and connector JSON Schemas (Redis, and each instance's memory)", "API Documentation"), - Constant.GLOSSARY_NAMESPACE -> ("Glossary items held in each instance's memory; bumping also rebuilds cached resource docs", "API Documentation") + Constant.GLOSSARY_NAMESPACE -> ("Glossary items held in each instance's memory; bumping also rebuilds cached resource docs", "API Documentation"), + Constant.DYNAMIC_ENTITY_NAMESPACE -> ("Dynamic entity definitions: the definition lists in Redis and each instance's definitions map", "Dynamic Entities") ) // Caches held in this instance's memory outside the shared in-memory store, which follow their // namespace's version through their own keys. Counted here so the page shows their real size. val inProcessEntries: Map[String, () => Long] = Map( Constant.MESSAGE_DOCS_NAMESPACE -> (() => code.api.v2_2_0.MessageDocsJsonCache.size + code.api.util.JsonSchemaGenerator.cacheSize), - Constant.GLOSSARY_NAMESPACE -> (() => code.api.util.Glossary.loadedItemCount.toLong) + Constant.GLOSSARY_NAMESPACE -> (() => code.api.util.Glossary.loadedItemCount.toLong), + Constant.DYNAMIC_ENTITY_NAMESPACE -> (() => code.api.dynamic.entity.helper.DynamicEntityHelper.keptDefinitionCount.toLong) ) var redisAvailable = true diff --git a/obp-api/src/test/scala/code/api/dynamic/entity/helper/DefinitionsCacheInvalidationTest.scala b/obp-api/src/test/scala/code/api/dynamic/entity/helper/DefinitionsCacheInvalidationTest.scala new file mode 100644 index 0000000000..516ee8e3e2 --- /dev/null +++ b/obp-api/src/test/scala/code/api/dynamic/entity/helper/DefinitionsCacheInvalidationTest.scala @@ -0,0 +1,197 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ + +package code.api.dynamic.entity.helper + +import code.api.Constant +import code.api.cache.Redis +import code.api.util.http4s.RequestScopeConnection +import code.setup.RedisTestTarget +import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} + +import java.util.concurrent.ConcurrentLinkedQueue + +/** + * A changed dynamic entity definition must reach every instance's definition caches. + * + * Two caches hold definitions: the definition lists NewStyle.getDynamicEntities memoizes in Redis + * (dynamicEntity.cache.ttl.seconds, default 30) and the definitions map each instance builds from them + * (dynamicEntity.definitions_map.cache.ttl.seconds, default 300). Both follow the dynamic_entity cache + * namespace version, which a definition change bumps once its transaction has committed. When a change + * only dropped this instance's map, the rebuild read the list memoized before the change and kept it for + * the map's whole time-to-live, and other instances never heard of the change at all: a new entity's + * record endpoints answered 404 for up to five minutes. + */ +class DefinitionsCacheInvalidationTest extends FlatSpec with Matchers with BeforeAndAfterEach { + + private val namespace = "dynamic_entity" + + private def version(): Long = Constant.getCacheNamespaceVersion(namespace) + + private def requireRedis(): Unit = + RedisTestTarget.requireReachable(Redis.isRedisReady, "the dynamic entity definition cache checks") + + private def runPending(pending: ConcurrentLinkedQueue[() => Unit]): Unit = + Iterator.continually(pending.poll()).takeWhile(_ != null).foreach(_.apply()) + + override def afterEach(): Unit = { + RequestScopeConnection.currentAfterCommit.remove() + super.afterEach() + } + + "The dynamic_entity namespace" should "be listed with the other cache namespaces" in { + // Listed, it shows on the cache info page and an administrator can bump it by hand. + Constant.ALL_CACHE_NAMESPACES should contain (namespace) + } + + "forgetDefinitions" should "bump the dynamic_entity version once the transaction commits" in { + // Until the commit the change is not visible to other connections: an instance that saw the bump + // earlier could read the old rows and cache them under the new version. + requireRedis() + val pending = new ConcurrentLinkedQueue[() => Unit]() + RequestScopeConnection.currentAfterCommit.set(pending) + val before = version() + + DynamicEntityHelper.forgetDefinitions() + + withClue("the version was bumped before the commit ") { + version() shouldBe before + } + runPending(pending) + withClue("the commit did not bump the dynamic_entity version, so no instance learns of the change ") { + version() shouldBe before + 1 + } + } + + it should "bump the dynamic resource-doc caches only once the transaction commits" in { + // Dynamic entities generate resource docs. Bumped before the commit, another instance could rebuild + // /resource-docs from the old definitions and keep that for the resource-doc time-to-live (an hour). + requireRedis() + val pending = new ConcurrentLinkedQueue[() => Unit]() + RequestScopeConnection.currentAfterCommit.set(pending) + val dynamicBefore = Constant.getCacheNamespaceVersion(Constant.RD_DYNAMIC_NAMESPACE) + val allBefore = Constant.getCacheNamespaceVersion(Constant.RD_ALL_NAMESPACE) + + DynamicEntityHelper.forgetDefinitions() + + withClue("the resource-doc caches were bumped before the commit ") { + Constant.getCacheNamespaceVersion(Constant.RD_DYNAMIC_NAMESPACE) shouldBe dynamicBefore + Constant.getCacheNamespaceVersion(Constant.RD_ALL_NAMESPACE) shouldBe allBefore + } + runPending(pending) + withClue("the commit did not bump the resource-doc caches, so /resource-docs keeps the old entities ") { + Constant.getCacheNamespaceVersion(Constant.RD_DYNAMIC_NAMESPACE) shouldBe dynamicBefore + 1 + Constant.getCacheNamespaceVersion(Constant.RD_ALL_NAMESPACE) shouldBe allBefore + 1 + } + } + + it should "bump it at once outside a transaction" in { + requireRedis() + val before = version() + DynamicEntityHelper.forgetDefinitions() + version() shouldBe before + 1 + } + + it should "leave it alone when the transaction rolls back" in { + requireRedis() + val pending = new ConcurrentLinkedQueue[() => Unit]() + RequestScopeConnection.currentAfterCommit.set(pending) + val before = version() + + DynamicEntityHelper.forgetDefinitions() + pending.clear() // what a rollback does with the after-commit actions + + version() shouldBe before + } + + "The definitions map cache" should "keep a built map while the version is unchanged" in { + val probe = new Probe + probe.cache.get shouldBe 1 + probe.cache.get shouldBe 1 + probe.builds shouldBe 1 + } + + it should "build it again once the version changes, as when another instance bumps it" in { + val probe = new Probe + probe.cache.get + probe.version += 1 + withClue("a map built under the previous version was served after the bump ") { + probe.cache.get shouldBe 2 + } + } + + it should "not keep a map built while the version changed under it" in { + // The build read the definitions before the change committed; it is kept under the version read + // before the build, which is already behind, so the next read builds again. + val probe = new Probe + probe.duringBuild = () => probe.version += 1 + probe.cache.get shouldBe 1 + probe.duringBuild = () => () + probe.cache.get shouldBe 2 + } + + it should "drop the map on forget even though the version did not change (Redis unavailable)" in { + val probe = new Probe + probe.cache.get + probe.cache.forget() + probe.cache.get shouldBe 2 + } + + it should "not keep a map whose build started before forget" in { + val probe = new Probe + probe.duringBuild = () => probe.cache.forget() + probe.cache.get shouldBe 1 + probe.duringBuild = () => () + probe.cache.get shouldBe 2 + } + + it should "keep nothing, and not read the version, when its time-to-live is zero" in { + val probe = new Probe(ttlMillis = 0L) + probe.cache.get shouldBe 1 + probe.cache.get shouldBe 2 + probe.versionReads shouldBe 0 + } + + it should "build it again once its time-to-live has passed" in { + val probe = new Probe(ttlMillis = 50L) + probe.cache.get + Thread.sleep(80) + probe.cache.get shouldBe 2 + } + + /** A map cache whose version and builds the test controls; each build returns its own number. */ + private class Probe(ttlMillis: Long = 60000L) { + @volatile var version: Long = 1L + @volatile var builds: Int = 0 + @volatile var versionReads: Int = 0 + @volatile var duringBuild: () => Unit = () => () + val cache = new DefinitionsMapCache[Int]( + ttlMillis, + () => { versionReads += 1; version }, + () => { builds += 1; duringBuild(); builds }) + } +}