chore(metrics): notification about slow graphql operations

package.json

@@ -1,6 +1,6 @@
 {
   "name": "hawk.api",
-  "version": "1.5.8",
+  "version": "1.5.9",
   "main": "index.ts",
   "license": "BUSL-1.1",
   "scripts": {

src/metrics/graphql.ts

@@ -2,6 +2,8 @@ import client from 'prom-client';
 import { ApolloServerPlugin, GraphQLRequestContext, GraphQLRequestListener } from 'apollo-server-plugin-base';
 import { GraphQLError } from 'graphql';
 import HawkCatcher from '@hawk.so/nodejs';
+import { notifySlowOperation } from './slowOperationAlert';
+import { buildGraphqlRequestContext, formatGraphqlErrorsForAlert } from './graphqlRequestDetails';
 /**
  * GraphQL operation duration histogram
  * Tracks GraphQL operation duration by operation name and type
@@ -93,6 +95,19 @@ export const graphqlMetricsPlugin: ApolloServerPlugin = {
           },
         });
 
+        notifySlowOperation(
+          `Slow GraphQL operation: ${operationType} ${operationName}`,
+          durationMs,
+          {
+            operationType,
+            operationName,
+            ...buildGraphqlRequestContext(ctx),
+            ...(hasErrors && {
+              errors: formatGraphqlErrorsForAlert(ctx.errors!),
+            }),
+          }
+        );
+
         // Track errors if any
         if (hasErrors) {
           ctx.errors!.forEach((error: GraphQLError) => {

src/metrics/graphqlRequestDetails.ts

@@ -0,0 +1,158 @@
+import { GraphQLRequestContext } from 'apollo-server-plugin-base';
+import { GraphQLError } from 'graphql';
+import { ResolverContextBase } from '../types/graphql';
+import { truncateText } from './slowOperationAlert';
+
+const MAX_ALERT_ERRORS = 10;
+const MAX_ALERT_ERRORS_LENGTH = 1200;
+
+const SENSITIVE_VARIABLE_KEYS = new Set([
+  'password',
+  'token',
+  'accesstoken',
+  'refreshtoken',
+  'secret',
+  'authorization',
+]);
+
+const HIGHLIGHTED_VARIABLE_KEYS = new Set([
+  'projectid',
+  'workspaceid',
+  'eventid',
+  'originaleventid',
+  'release',
+  'search',
+  'assignee',
+  'cursor',
+]);
+
+/**
+ * Redact sensitive GraphQL variables before sending alerts.
+ *
+ * @param value - variable value
+ * @param key - variable key
+ * @returns sanitized value
+ */
+function sanitizeVariableValue(value: unknown, key: string): unknown {
+  if (SENSITIVE_VARIABLE_KEYS.has(key.toLowerCase())) {
+    return '[redacted]';
+  }
+
+  if (Array.isArray(value)) {
+    return value.map((item, index) => sanitizeVariableValue(item, `${key}[${index}]`));
+  }
+
+  if (value && typeof value === 'object') {
+    // eslint-disable-next-line @typescript-eslint/no-use-before-define
+    return sanitizeVariables(value as Record<string, unknown>);
+  }
+
+  return value;
+}
+
+/**
+ * Redact sensitive GraphQL variables before sending alerts.
+ *
+ * @param variables - GraphQL request variables
+ * @returns sanitized variables
+ */
+function sanitizeVariables(
+  variables: Record<string, unknown> | null | undefined
+): Record<string, unknown> {
+  /**
+   * Null / non-object values are treated as empty — many clients send
+   * `variables: null` for operations without variables, and arrays are not a
+   * valid GraphQL variables map.
+   */
+  if (variables == null || typeof variables !== 'object' || Array.isArray(variables)) {
+    return {};
+  }
+
+  return Object.fromEntries(
+    Object.entries(variables).map(([key, value]) => [key, sanitizeVariableValue(value, key)])
+  );
+}
+
+/**
+ * Extract useful identifiers from nested GraphQL variables.
+ *
+ * @param value - variable value
+ * @param prefix - nested path prefix
+ * @param result - accumulator for extracted ids
+ * @returns extracted identifiers
+ */
+function collectHighlightedIds(
+  value: unknown,
+  prefix = '',
+  result: Record<string, string | number | boolean> = {}
+): Record<string, string | number | boolean> {
+  if (!value || typeof value !== 'object') {
+    return result;
+  }
+
+  for (const [key, nestedValue] of Object.entries(value as Record<string, unknown>)) {
+    const path = prefix ? `${prefix}.${key}` : key;
+
+    if (
+      HIGHLIGHTED_VARIABLE_KEYS.has(key.toLowerCase()) &&
+      (typeof nestedValue === 'string' || typeof nestedValue === 'number' || typeof nestedValue === 'boolean')
+    ) {
+      result[path] = nestedValue;
+      continue;
+    }
+
+    if (nestedValue && typeof nestedValue === 'object' && !Array.isArray(nestedValue)) {
+      collectHighlightedIds(nestedValue, path, result);
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Flatten GraphQL errors into a capped string for Hawk alert context.
+ * sanitizeContext() only truncates top-level strings, not values nested in arrays.
+ * Reserves space for an omitted-count suffix and truncateText()'s ellipsis.
+ *
+ * @param errors - GraphQL errors from the request
+ * @returns flattened and truncated error messages
+ */
+export function formatGraphqlErrorsForAlert(errors: readonly GraphQLError[]): string {
+  const messages = errors.slice(0, MAX_ALERT_ERRORS).map((error) => error.message);
+  const omittedCount = errors.length - messages.length;
+  const omittedSuffix = omittedCount > 0 ? `; …(+${omittedCount} more)` : '';
+  const maxMessagesLength = Math.max(0, MAX_ALERT_ERRORS_LENGTH - omittedSuffix.length - 1);
+
+  return `${truncateText(messages.join('; '), maxMessagesLength)}${omittedSuffix}`;
+}
+
+/**
+ * Build request context for slow GraphQL operation alerts.
+ *
+ * @param ctx - GraphQL request context
+ * @returns alert context
+ */
+export function buildGraphqlRequestContext(ctx: GraphQLRequestContext): Record<string, unknown> {
+  const context = ctx.context as ResolverContextBase | undefined;
+  const variables = sanitizeVariables(
+    ctx.request.variables as Record<string, unknown> | null | undefined
+  );
+  const highlightedIds = collectHighlightedIds(variables);
+  const alertContext: Record<string, unknown> = {};
+
+  if (context?.user?.id) {
+    alertContext.userId = context.user.id;
+  }
+
+  if (Object.keys(highlightedIds).length > 0) {
+    alertContext.ids = highlightedIds;
+  }
+
+  const variablesJson = JSON.stringify(variables);
+
+  if (variablesJson && variablesJson !== '{}') {
+    alertContext.variables = truncateText(variablesJson, 1200);
+  }
+
+  return alertContext;
+}

src/metrics/mongodb.ts

@@ -2,6 +2,7 @@ import promClient from 'prom-client';
 import { MongoClient, MongoClientOptions } from 'mongodb';
 import { Effect, sgr } from '../utils/ansi';
 import HawkCatcher from '@hawk.so/nodejs';
+import { notifySlowOperation, truncateText } from './slowOperationAlert';
 
 /**
  * MongoDB command duration histogram
@@ -156,6 +157,7 @@ function colorizeDuration(duration: number): string {
  */
 interface StoredCommandInfo {
   formattedCommand: string;
+  plainFormattedCommand: string;
   timestamp: number;
 }
 
@@ -202,7 +204,8 @@ setInterval(cleanupStaleCommandInfo, COMMAND_INFO_TIMEOUT_MS);
  */
 function storeCommandInfo(event: any): void {
   const collectionRaw = extractCollectionFromCommand(event.command, event.commandName);
-  const collection = sgr(normalizeCollectionName(collectionRaw), Effect.ForegroundGreen);
+  const collectionName = normalizeCollectionName(collectionRaw);
+  const collection = sgr(collectionName, Effect.ForegroundGreen);
   const db = event.databaseName || 'unknown db';
   const commandName = sgr(event.commandName, Effect.ForegroundRed);
   const filter = event.command.filter;
@@ -212,11 +215,12 @@ function storeCommandInfo(event: any): void {
   const params = filter || update || pipeline;
   const paramsStr = formatParams(params);
   const projectionStr = projection ? ` projection: ${formatParams(projection)}` : '';
-
+  const plainFormattedCommand = `[${event.requestId}] ${db}.${collectionName}.${event.commandName}(${paramsStr})${projectionStr}`;
   const formattedCommand = `[${event.requestId}] ${db}.${collection}.${commandName}(${paramsStr})${projectionStr}`;
 
   commandInfoMap.set(event.requestId, {
     formattedCommand,
+    plainFormattedCommand,
     timestamp: Date.now(),
   });
 }
@@ -232,9 +236,24 @@ function logCommandSucceeded(event: any): void {
 
   if (info) {
     console.log(`${info.formattedCommand} ✓ ${durationStr}`);
+    notifySlowOperation(
+      `Slow MongoDB command: ${event.commandName}`,
+      event.duration,
+      {
+        requestId: event.requestId,
+        command: truncateText(info.plainFormattedCommand),
+      }
+    );
     commandInfoMap.delete(event.requestId);
   } else {
     console.log(`[${event.requestId}] ${event.commandName} ✓ ${durationStr}`);
+    notifySlowOperation(
+      `Slow MongoDB command: ${event.commandName}`,
+      event.duration,
+      {
+        requestId: event.requestId,
+      }
+    );
   }
 }
 
@@ -250,9 +269,26 @@ function logCommandFailed(event: any): void {
 
   if (info) {
     console.error(`${info.formattedCommand} ✗ ${errorMsg} ${durationStr}`);
+    notifySlowOperation(
+      `Slow MongoDB command: ${event.commandName}`,
+      event.duration,
+      {
+        requestId: event.requestId,
+        command: truncateText(info.plainFormattedCommand),
+        error: truncateText(errorMsg, 500),
+      }
+    );
     commandInfoMap.delete(event.requestId);
   } else {
     console.error(`[${event.requestId}] ${event.commandName} ✗ ${errorMsg} ${durationStr}`);
+    notifySlowOperation(
+      `Slow MongoDB command: ${event.commandName}`,
+      event.duration,
+      {
+        requestId: event.requestId,
+        error: truncateText(errorMsg, 500),
+      }
+    );
   }
 }
 

src/metrics/slowOperationAlert.ts

@@ -0,0 +1,69 @@
+import HawkCatcher from '@hawk.so/nodejs';
+
+export const SLOW_OPERATION_THRESHOLD_MS = 10000;
+const MAX_CONTEXT_STRING_LENGTH = 2500;
+
+/**
+ * Truncate text for slow operation context fields.
+ *
+ * @param value - text to truncate
+ * @param maxLength - max allowed length
+ * @returns truncated text
+ */
+function truncateText(value: string, maxLength = MAX_CONTEXT_STRING_LENGTH): string {
+  if (value.length <= maxLength) {
+    return value;
+  }
+
+  return `${value.slice(0, maxLength)}…`;
+}
+
+/**
+ * Truncate long string values in alert context.
+ *
+ * @param context - alert context
+ * @returns sanitized context
+ */
+function sanitizeContext(context: Record<string, unknown>): Record<string, unknown> {
+  return Object.fromEntries(
+    Object.entries(context).map(([key, value]) => {
+      if (typeof value === 'string') {
+        return [key, truncateText(value)];
+      }
+
+      return [key, value];
+    })
+  );
+}
+
+/**
+ * Send slow operation alert to Hawk via HawkCatcher.
+ *
+ * @param message - short alert message
+ * @param durationMs - operation duration in milliseconds
+ * @param context - additional alert context
+ */
+export function notifySlowOperation(
+  message: string,
+  durationMs: number,
+  context: Record<string, unknown> = {}
+): void {
+  if (
+    process.env.NODE_ENV === 'test' ||
+    process.env.NODE_ENV === 'e2e' ||
+    durationMs < SLOW_OPERATION_THRESHOLD_MS
+  ) {
+    return;
+  }
+
+  try {
+    HawkCatcher.send(new Error(message), {
+      durationMs,
+      ...sanitizeContext(context),
+    });
+  } catch (error) {
+    console.log('Couldn\'t send slow operation alert to Hawk', error);
+  }
+}
+
+export { truncateText };