fix(dynamic-entity): drop the memoized definition lists when definitions change

obp-api/src/main/scala/code/api/constant/constant.scala

@@ -374,6 +374,10 @@ object Constant extends MdcLoggable {
   // The Glossary as each instance holds it in memory. Bumping it reloads the Glossary at once and
   // rebuilds every cached resource-docs document, because their keys carry the Glossary version.
   final val GLOSSARY_NAMESPACE = "glossary"
+  // Dynamic entity definitions: the definition lists memoized in Redis and the definitions map each
+  // instance builds from them. A definition change bumps it once committed, so every instance reads
+  // the definitions again.
+  final val DYNAMIC_ENTITY_NAMESPACE = "dynamic_entity"
 
   // List of all versioned cache namespaces
   final val ALL_CACHE_NAMESPACES = List(
@@ -393,7 +397,8 @@ object Constant extends MdcLoggable {
     FINANCIAL_PRODUCTS_NAMESPACE,
     API_PRODUCTS_NAMESPACE,
     MESSAGE_DOCS_NAMESPACE,
-    GLOSSARY_NAMESPACE
+    GLOSSARY_NAMESPACE,
+    DYNAMIC_ENTITY_NAMESPACE
   )
 
   // Cache key prefixes with global namespace and versioning for easy invalidation

obp-api/src/main/scala/code/api/dynamic/entity/helper/DefinitionsMapCache.scala

@@ -0,0 +1,68 @@
+/**
+Open Bank Project - API
+Copyright (C) 2011-2026, TESOBE GmbH.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Email: contact@tesobe.com
+TESOBE GmbH.
+Osloer Strasse 16/17
+Berlin 13359, Germany
+
+This product includes software developed at
+TESOBE (http://www.tesobe.com/)
+
+  */
+
+package code.api.dynamic.entity.helper
+
+/**
+ * Keeps one built value (the dynamic entity definitions map) for up to `ttlMillis`, and only while
+ * `currentVersion` still returns the version it was built under.
+ *
+ * The version is read before building, so a value built while the version changed is kept under the
+ * old version and dropped on the next read. [[forget]] drops the value on this instance alone, for when
+ * the version cannot change (Redis unavailable); a build that started before it is not kept after it.
+ * Nothing is kept, and the version is not read, when `ttlMillis` is not positive.
+ */
+private[helper] final class DefinitionsMapCache[M](ttlMillis: Long, currentVersion: () => Long, build: () => M) {
+
+  // (built at, version it was built under, value)
+  @volatile private var entry: Option[(Long, Long, M)] = None
+  // Replaced by forget; a build keeps its value only if the token it started under is still current.
+  @volatile private var token: AnyRef = new Object
+
+  def get: M =
+    if (ttlMillis <= 0) build()
+    else {
+      val now = System.currentTimeMillis()
+      val version = currentVersion()
+      entry match {
+        case Some((builtAt, builtUnder, value)) if builtUnder == version && now - builtAt < ttlMillis => value
+        case _ =>
+          val startedUnder = token
+          val value = build()
+          if (token eq startedUnder) entry = Some((now, version, value))
+          value
+      }
+    }
+
+  def forget(): Unit = {
+    token = new Object
+    entry = None
+  }
+
+  /** The value kept now, if any, whatever its version. */
+  def kept: Option[M] = entry.map(_._3)
+}

obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala

@@ -27,6 +27,7 @@ TESOBE (http://www.tesobe.com/)
 
 package code.api.dynamic.entity.helper
 
+import code.api.Constant
 import code.api.util.APIUtil.{EmptyBody, ResourceDoc, userAuthenticationMessage}
 import code.api.util.ApiRole.getOrCreateDynamicApiRole
 import code.api.util.ApiTag._
@@ -202,35 +203,44 @@ object DynamicEntityHelper {
   // It is asked for many times per request (the access checks, each field's read check, the resource
   // docs), and building it reads every definition and parses each one's JSON: with 120 definitions
   // that was ~5 ms a time, and a Dynamic Query returning 11 records spent ~0.5 s rebuilding it. So
-  // the built map is kept for dynamicEntity.definitions_map.cache.ttl.seconds (default 300), and
-  // forgotten the moment a definition is created, updated or deleted on this node
-  // (MappedDynamicEntityProvider calls forgetDefinitions). Another node sees such a change when its
-  // copy expires. Not kept in test mode, where a test may change the table behind the provider.
+  // the built map is kept for dynamicEntity.definitions_map.cache.ttl.seconds (default 300), and only
+  // while the dynamic_entity cache namespace version is the one it was built under: a definition change
+  // bumps that version once committed (forgetDefinitions), and every instance, which re-reads the version
+  // at most once a second, then builds the map again. Not kept in test mode, where a test may change the
+  // table behind the provider.
   private val definitionsMapTtlMillis: Long =
     if (net.liftweb.util.Props.testMode) 0L
     else APIUtil.getPropsAsLongValue("dynamicEntity.definitions_map.cache.ttl.seconds", 300L) * 1000L
-  // (built at, map). Each change replaces the token, and a build keeps its map only while the token it
-  // started under is still the current one, so a build that started before a change is not kept after it.
-  @volatile private var definitionsMapCache: Option[(Long, Map[(String, String), DynamicEntityInfo])] = None
-  @volatile private var definitionsToken: AnyRef = new Object
-
-  /** Forget the kept definitions map, so the next use reads the definitions again. */
-  def forgetDefinitions(): Unit = {
-    definitionsToken = new Object
-    definitionsMapCache = None
-  }
 
-  def definitionsMap: Map[(String, String), DynamicEntityInfo] = {
-    val now = System.currentTimeMillis()
-    definitionsMapCache match {
-      case Some((builtAt, map)) if now - builtAt < definitionsMapTtlMillis => map
-      case _ =>
-        val token = definitionsToken
-        val map = buildDefinitionsMap
-        if (definitionsMapTtlMillis > 0 && (definitionsToken eq token)) definitionsMapCache = Some((now, map))
-        map
+  private val definitionsMapCache = new DefinitionsMapCache[Map[(String, String), DynamicEntityInfo]](
+    definitionsMapTtlMillis,
+    () => Constant.recentCacheNamespaceVersion(Constant.DYNAMIC_ENTITY_NAMESPACE),
+    () => buildDefinitionsMap)
+
+  /**
+   * Tell every instance that a definition was created, updated or deleted, once the request's transaction
+   * has committed (at once outside a transaction): bump the dynamic_entity cache namespace version, which
+   * the definitions map and the definition lists NewStyle.getDynamicEntities memoizes in Redis both carry,
+   * so all of them read the definitions again, and then the dynamic resource-doc caches, which hold the
+   * docs generated from the definitions. Not earlier, because until the commit the change is not visible
+   * to other connections: an instance that saw a bump first could read the old rows and keep them under the
+   * new version. Nothing happens if the transaction rolls back.
+   *
+   * This instance also drops its map directly, which is what keeps it current when Redis is unavailable
+   * and the version cannot change. Another instance follows the dynamic_entity version within a second, so
+   * for that second it can still build resource docs from its previous map.
+   */
+  def forgetDefinitions(): Unit =
+    code.api.util.http4s.RequestScopeConnection.afterCommit {
+      Constant.incrementCacheNamespaceVersion(Constant.DYNAMIC_ENTITY_NAMESPACE)
+      definitionsMapCache.forget()
+      NewStyle.function.invalidateDynamicResourceDocCaches()
     }
-  }
+
+  def definitionsMap: Map[(String, String), DynamicEntityInfo] = definitionsMapCache.get
+
+  /** The number of definitions in this instance's kept map, for the cache info page. */
+  def keptDefinitionCount: Int = definitionsMapCache.kept.map(_.size).getOrElse(0)
 
   private def buildDefinitionsMap: Map[(String, String), DynamicEntityInfo] = NewStyle.function.getDynamicEntities(None, true).map(it => ((DynamicEntitySpace.bankIdOrSystem(it.bankId), it.entityName), DynamicEntityInfo(it.metadataJson, it.entityName, it.bankId, it.hasPersonalEntity, it.hasPublicAccess, it.hasCommunityAccess, it.personalRequiresRole, it.useRowLevelAccess, it.authMode))).toMap
 

obp-api/src/main/scala/code/api/util/NewStyle.scala

@@ -3543,12 +3543,14 @@
      * next `/resource-docs` request regenerates from the database instead of
      * serving a pre-change snapshot (TTL is 1 hour by default).
      *
-     * Call after a Dynamic Entity, Dynamic Endpoint or Dynamic Resource Doc is
-     * created, updated, or deleted: each one generates resource docs. The static
-     * resource-doc / swagger caches are not touched because dynamic docs only
-     * reach them through the content=all documents, which are left to their TTL.
+     * Call after a Dynamic Endpoint or Dynamic Resource Doc is created, updated,
+     * or deleted: each one generates resource docs. For a Dynamic Entity,
+     * DynamicEntityHelper.forgetDefinitions calls it once the change has
+     * committed. The static resource-doc / swagger caches are not touched because
+     * dynamic docs only reach them through the content=all documents, which are
+     * left to their TTL.
      */
-    private def invalidateDynamicResourceDocCaches(): Unit = {
+    private[code] def invalidateDynamicResourceDocCaches(): Unit = {
       Constant.incrementCacheNamespaceVersion(Constant.RD_DYNAMIC_NAMESPACE)
       Constant.incrementCacheNamespaceVersion(Constant.RD_ALL_NAMESPACE)
     }
@@ -3589,9 +3591,9 @@
       }
 
       Future {
-        val result = DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate(dynamicEntity)
-        if (result.isDefined) invalidateDynamicResourceDocCaches()
-        result
+        // The provider calls DynamicEntityHelper.forgetDefinitions, which also drops the dynamic
+        // resource-doc caches once the change has committed.
+        DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate(dynamicEntity)
       }
     }
 
@@ -3630,9 +3632,9 @@
       }
 
       Future {
-        val result = DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate(dynamicEntity)
-        if (result.isDefined) invalidateDynamicResourceDocCaches()
-        result
+        // The provider calls DynamicEntityHelper.forgetDefinitions, which also drops the dynamic
+        // resource-doc caches once the change has committed.
+        DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate(dynamicEntity)
       }
 
     }
@@ -3674,8 +3676,8 @@
           }
           deleteEntitleMentResult
         }
-        // The entity (and its generated resource docs) are gone — drop the stale doc caches.
-        if (result.exists(deleted => deleted)) invalidateDynamicResourceDocCaches()
+        // The entity's generated resource docs go with it: the provider's delete calls
+        // DynamicEntityHelper.forgetDefinitions, which drops the doc caches once the delete has committed.
         result
       }
     }
@@ -3704,9 +3706,27 @@
     }
 
     def getDynamicEntities(bankId: Option[String], returnBothBankAndSystemLevel: Boolean): List[DynamicEntityT] = {
-      import scala.concurrent.duration._
-
       validateBankId(bankId, None)
+      getDynamicEntities(bankId, returnBothBankAndSystemLevel, dynamicEntityDefinitionsVersion)
+    }
+
+    /**
+     * The dynamic_entity cache namespace version, which the memoized definition lists below carry in their
+     * keys. DynamicEntityHelper.forgetDefinitions bumps it once a definition change has committed, so every
+     * list memoized before the change becomes unreachable on every instance, including one that a read still
+     * in flight writes after the bump: it is written under the version that read started with.
+     */
+    private def dynamicEntityDefinitionsVersion: Long =
+      if (dynamicEntityTTL == 0) 0L // nothing is memoized, so no need to ask Redis
+      else Constant.recentCacheNamespaceVersion(Constant.DYNAMIC_ENTITY_NAMESPACE)
+
+    // The version is a parameter so that CacheKeyFromArguments renders it into the key; the name stays
+    // getDynamicEntities so the key and its Telemetry label stay the same apart from the version. Keep
+    // buildCacheKey as the tail expression (see getEndpointMappingsCached): bound to a val first, the macro
+    // renders no arguments and every list would share one key whatever the version. Test mode memoizes
+    // nothing, so no test would notice; the rendering was checked with javap.
+    private def getDynamicEntities(bankId: Option[String], returnBothBankAndSystemLevel: Boolean, definitionsVersion: Long): List[DynamicEntityT] = {
+      import scala.concurrent.duration._
 
       var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString)
       CacheKeyFromArguments.buildCacheKey {
@@ -3715,8 +3735,11 @@
         }
       }
     }
-
-    def getDynamicEntitiesByUserId(userId: String): List[DynamicEntityT] = {
+
+    def getDynamicEntitiesByUserId(userId: String): List[DynamicEntityT] =
+      getDynamicEntitiesByUserId(userId, dynamicEntityDefinitionsVersion)
+
+    private def getDynamicEntitiesByUserId(userId: String, definitionsVersion: Long): List[DynamicEntityT] = {
       import scala.concurrent.duration._
 
       var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString)

obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala

@@ -2444,14 +2444,16 @@ object JSONFactory600 extends CustomJsonFormats with MdcLoggable {
       Constant.FINANCIAL_PRODUCTS_NAMESPACE -> ("Financial product list (bank-scoped and all-banks)", "Products"),
       Constant.API_PRODUCTS_NAMESPACE -> ("Api product list (all banks)", "Products"),
       Constant.MESSAGE_DOCS_NAMESPACE -> ("Message docs and connector JSON Schemas (Redis, and each instance's memory)", "API Documentation"),
-      Constant.GLOSSARY_NAMESPACE -> ("Glossary items held in each instance's memory; bumping also rebuilds cached resource docs", "API Documentation")
+      Constant.GLOSSARY_NAMESPACE -> ("Glossary items held in each instance's memory; bumping also rebuilds cached resource docs", "API Documentation"),
+      Constant.DYNAMIC_ENTITY_NAMESPACE -> ("Dynamic entity definitions: the definition lists in Redis and each instance's definitions map", "Dynamic Entities")
     )
 
     // Caches held in this instance's memory outside the shared in-memory store, which follow their
     // namespace's version through their own keys. Counted here so the page shows their real size.
     val inProcessEntries: Map[String, () => Long] = Map(
       Constant.MESSAGE_DOCS_NAMESPACE -> (() => code.api.v2_2_0.MessageDocsJsonCache.size + code.api.util.JsonSchemaGenerator.cacheSize),
-      Constant.GLOSSARY_NAMESPACE -> (() => code.api.util.Glossary.loadedItemCount.toLong)
+      Constant.GLOSSARY_NAMESPACE -> (() => code.api.util.Glossary.loadedItemCount.toLong),
+      Constant.DYNAMIC_ENTITY_NAMESPACE -> (() => code.api.dynamic.entity.helper.DynamicEntityHelper.keptDefinitionCount.toLong)
     )
 
     var redisAvailable = true