Allow custom database indexes via environment variables

  Closes #206.

Author: KtorZ

CHANGELOG.md

@@ -6,6 +6,12 @@
 
 - New slot-range index usage for wildcard (*) match queries (see [#194](https://github.com/CardanoSolutions/kupo/issues/194)).
 
+- Custom indexes can now be added via environment variable, prefixed with `KUPO_INDEX_`. They behave as standard indexes with regards to `--defer-db-indexes` and are only installed if not already present. Indexes are named after the environment variable (after the dropping the prefix). So for example, to add an index for unspent inputs by payment credential, you can now provide:
+
+  ```console
+  KUPO_INDEX_inputsByPaymentCredentialUnspent="inputs(payment_credential COLLATE NOCASE) WHERE spent_at IS NULL" kupo ...
+  ```
+
 #### Changed
 
 - Bumped internal dependencies to cardano-node==11.0.1.

docs/api/nightly.yaml

@@ -273,6 +273,25 @@ info:
     $ man kupo
     ```
 
+    # Custom database indexes
+
+    Kupo supports custom database indexes, provided by environment variable.
+    Hence, any environment variable prefixed with `KUPO_INDEX_` will be parsed
+    and interpreted as a custom database index, managed with the pre-defined
+    indexes.
+
+    For example, you can add a custom index `inputsByPaymentCredentialUnspent`
+    for matching unspent inputs by payment cred by setting the following
+    environment variable:
+
+    ```env
+    KUPO_INDEX_inputsByPaymentCredentialUnspent="inputs(payment_credential COLLATE NOCASE) WHERE spent_at IS NULL"
+    ```
+
+    Use this carefully, as too many indexes may greatly impact the database sync performances and also other queries.
+
+    Existing indexes and the overall database internal schema is [available here](https://github.com/CardanoSolutions/kupo/tree/master/db#overview).
+
     # Patterns
 
     ## Overview

src/Kupo/App/Database/SQLite.hs

@@ -181,6 +181,9 @@ import Kupo.Control.MonadLog
     ( TraceProgress (..)
     , nullTracer
     )
+import System.Environment
+    ( getEnvironment
+    )
 import Text.URI
     ( URI
     )
@@ -1161,33 +1164,45 @@ installIndexes
     -> Connection
     -> DeferIndexesInstallation
     -> IO ()
-installIndexes tr conn = \case
-    SkipNonEssentialIndexes -> do
-        dropIndexIfExists (contramap DatabaseConnection tr) conn "inputsByAddress" False
-        dropIndexIfExists (contramap DatabaseConnection tr) conn "inputsByDatumHash" False
-        dropIndexIfExists (contramap DatabaseConnection tr) conn "inputsByPaymentCredential" False
-        dropIndexIfExists (contramap DatabaseConnection tr) conn "inputsByCreatedAt" False
-        dropIndexIfExists (contramap DatabaseConnection tr) conn "inputsBySpentAt" False
-        dropIndexIfExists (contramap DatabaseConnection tr) conn "policiesByPolicyId" False
-    InstallIndexesIfNotExist -> do
-        installIndex tr conn
-            "inputsByAddress"
-            "inputs(address COLLATE NOCASE)"
-        installIndex tr conn
-            "inputsByDatumHash"
-            "inputs(datum_hash)"
-        installIndex tr conn
-            "inputsByPaymentCredential"
-            "inputs(payment_credential COLLATE NOCASE)"
-        installIndex tr conn
-            "inputsByCreatedAt"
-            "inputs(created_at)"
-        installIndex tr conn
-            "inputsBySpentAt"
-            "inputs(spent_at)"
-        installIndex tr conn
-            "policiesByPolicyId"
-            "policies(policy_id)"
+installIndexes tr conn indexBehavior = do
+    customIndexes <- parseCustomIndexes <$> getEnvironment
+
+    case indexBehavior of
+        SkipNonEssentialIndexes -> do
+            dropIndex "inputsByAddress"
+            dropIndex "inputsByDatumHash"
+            dropIndex "inputsByPaymentCredential"
+            dropIndex "inputsByCreatedAt"
+            dropIndex "inputsBySpentAt"
+            dropIndex "policiesByPolicyId"
+            forM_ customIndexes (\(name, _definition) -> dropIndex name)
+          where
+            dropIndex name = dropIndexIfExists (contramap DatabaseConnection tr) conn name False
+        InstallIndexesIfNotExist -> do
+            installIndex tr conn
+                "inputsByAddress"
+                "inputs(address COLLATE NOCASE)"
+            installIndex tr conn
+                "inputsByDatumHash"
+                "inputs(datum_hash)"
+            installIndex tr conn
+                "inputsByPaymentCredential"
+                "inputs(payment_credential COLLATE NOCASE)"
+            installIndex tr conn
+                "inputsByCreatedAt"
+                "inputs(created_at)"
+            installIndex tr conn
+                "inputsBySpentAt"
+                "inputs(spent_at)"
+            installIndex tr conn
+                "policiesByPolicyId"
+                "policies(policy_id)"
+            forM_ customIndexes (uncurry (installIndex tr conn))
+  where
+    parseCustomIndexes = mapMaybe $ \(key, value) ->
+        case T.stripPrefix "KUPO_INDEX_" (toText key) of
+          Nothing -> Nothing
+          Just name -> Just (name, toText value)
 
 -- Create the given index with some extra logging around it.
 installIndex :: Tracer IO TraceDatabase -> Connection -> Text -> Text -> IO ()