SecureChatAI.php

<?php

namespace Stanford\SecureChatAI;

require_once "emLoggerTrait.php";
require_once "classes/SecureChatLog.php";
require_once "classes/Models/ModelInterface.php";
require_once "classes/Models/BaseModelRequest.php";
require_once "classes/Models/GPTModelRequest.php";
require_once "classes/Models/WhisperModelRequest.php";
require_once "classes/Models/GeminiModelRequest.php";
require_once "classes/Models/ClaudeModelRequest.php";
require_once "classes/Models/GenericModelRequest.php";
require_once "classes/Models/GPT4oMiniTTSModelRequest.php";

require_once "classes/ToolInterface.php";
require_once "classes/ToolUse.php";
require_once "classes/ToolResult.php";
require_once "classes/ToolContext.php";
require_once "classes/AbortController.php";
require_once "classes/HookInterface.php";
require_once "classes/HookResult.php";
require_once "classes/ProjectAccessPreHook.php";
require_once "classes/JsonConfigToolAdapter.php";
require_once "classes/ToolPipeline.php";
require_once "classes/TokenPolicy.php";
require_once "classes/TokenPolicyPreHook.php";

require_once __DIR__ . '/vendor/autoload.php';

use Google\Exception;
use GuzzleHttp\Client;
use Yethee\Tiktoken\EncoderProvider;

class SecureChatAI extends \ExternalModules\AbstractExternalModule
{
    use emLoggerTrait;

    /**
     * Models that support the json_schema response format, in PREFERENCE ORDER.
     *
     * Two consumers, and only one of them cares about the order:
     *   - filterDefaultParamsForModel(): membership test only (order irrelevant)
     *   - runAgentLoop(): takes the FIRST entry present in api-settings as the
     *     fallback when the requested model can't do schemas. Whatever sits at the
     *     front is what every agent-mode turn silently runs on.
     *
     * gpt-4-1 used to be first, so it was always the pick — dated, and the lowest
     * Azure quota of the group (1000/2000 vs gpt-5-4's 1500/3000). Ordering is now
     * newest full models, then older full models, then reasoning models, with the
     * nanos last: cheapest quota but weakest at the multi-step tool reasoning agent
     * mode depends on.
     *
     * Was duplicated as a local in both methods; a capability list that drifts
     * between copies is a silent "Unsupported model" mid-loop, so it lives here.
     */
    private const SCHEMA_CAPABLE_MODELS = [
        'gpt-5-4', 'gpt-5-6-sol', 'gpt-5-6-luna', 'gpt-5-6-terra', 'gpt-5',
        'gpt-4-1',
        'o3', 'o4-mini', 'o1', 'o3-mini',
        'gpt-5-4-nano', 'gpt-4-1-nano',
    ];

    private array $defaultParams;
    private $guzzleClient = null;
    private $guzzleTimeout = 5.0;
    /** [token policy => [model alias => config]], see TokenPolicy::buildRegistry() */
    private array $modelConfig = [];
    /**
     * Policy of the callAI() currently executing on this instance, or null.
     * getModuleInstance() hands every caller in a request the same instance, so
     * a nested callAI() (e.g. a tool EM calling back during an agent loop) uses
     * this as its minimum and can never drop below the enclosing call's policy.
     */
    private ?string $activeTokenPolicy = null;
    private bool $initialized = false;
    private ?EncoderProvider $encoderProvider = null;

    public function __construct()
    {
        parent::__construct();
    }

    private function initSecureChatAI()
    {
        if ($this->initialized) return;
        // Set default LLM model parameters
        $this->defaultParams = [
            'temperature' => (float)$this->getSystemSetting('gpt-temperature') ?: 0.7,
            'top_p' => (float)$this->getSystemSetting('gpt-top-p') ?: 0.9,
            'frequency_penalty' => (float)$this->getSystemSetting('gpt-frequency-penalty') ?: 0.5,
            'presence_penalty' => (float)$this->getSystemSetting('gpt-presence-penalty') ?: 0,
            'max_tokens' => (int)$this->getSystemSetting('gpt-max-tokens') ?: 16384,
            'reasoning_effort' => $this->getSystemSetting('reasoning-effort'),
            'stop' => null,
            'model' => 'gpt-4o'
        ];

        // Initialize the model configurations from system settings, keyed by
        // token policy then alias. Each row's token is resolved here (row
        // override, else the system token for the row's policy) and never
        // changed per call.
        $this->modelConfig = TokenPolicy::buildRegistry(
            $this->framework->getSubSettings('api-settings') ?: [],
            (string)($this->getSystemSetting('aihub-token-standard') ?? ''),
            (string)($this->getSystemSetting('aihub-token-high-threshold') ?? '')
        );
        foreach ($this->modelConfig as $configs) {
            foreach ($configs as $modelAlias => $config) {
                if ($config['is_default']) {
                    $this->defaultParams['model'] = $modelAlias;
                }
            }
        }

        // Set Guzzle info
        $timeout = $this->getSystemSetting('guzzle-timeout') ? (float)(strip_tags($this->getSystemSetting('guzzle-timeout'))) : $this->getGuzzleTimeout();
        $this->setGuzzleTimeout($timeout);
        $this->guzzleClient = $this->getGuzzleClient();

        $this->initialized = true;
    }

    public function getSecureChatLogs($offset)
    {
        $offset = intval($offset);
        return SecureChatLog::getAllLogs($this, $offset);
    }

    /**
     * Get logs for a specific session ID
     * @param string $session_id
     * @param int|null $project_id Optional project filter
     * @return array
     */
    public function getSecureChatLogsBySession($session_id, $project_id = null)
    {
        return SecureChatLog::getLogsBySession($this, $session_id, $project_id);
    }

    /**
     * Get logs for a specific project.
     * Fetches all logs and filters by the project_id stored in the JSON message payload,
     * since the EM framework's project_id column may not match (logs written from system context).
     * @param int $project_id
     * @param int $offset
     * @return array Decoded log data arrays (same format as SecureChatLog::getLog())
     */
    public function getProjectSecureChatLogs($project_id, $offset)
    {
        // Cannot use SecureChatLog or ASEMLO here because $module->queryLogs()
        // auto-filters by project_id when called from project context. Logs written
        // from system context have NULL in the EM project_id column even though the
        // JSON payload contains the correct project_id. We bypass the EM framework
        // entirely and load raw data.
        $pid = intval($project_id);
        $offset = intval($offset);
        $sql = "SELECT log_id, timestamp, record, message
                FROM redcap_external_modules_log
                WHERE external_module_id = (
                    SELECT external_module_id FROM redcap_external_modules
                    WHERE directory_prefix = ?
                    LIMIT 1
                )
                AND record IN ('SecureChatLog', 'SecureChatLogError')
                AND (message LIKE ? OR message LIKE ?)
                ORDER BY log_id DESC
                LIMIT 1000 OFFSET ?";
        $pattern = '%"project_id":' . $pid . '%';
        $patternSpaced = '%"project_id": ' . $pid . '%';
        $prefix = $this->PREFIX;
        $result = $this->query($sql, [$prefix, $pattern, $patternSpaced, $offset]);

        $logs = [];
        while ($row = $result->fetch_assoc()) {
            $decoded = json_decode($row['message'], true);
            if ($decoded === null) continue;
            // Verify project_id actually matches (LIKE is broad)
            $logPid = $decoded['project_id'] ?? null;
            if (intval($logPid) !== $pid) continue;
            $decoded['timestamp'] = $row['timestamp'];
            $decoded['id'] = $row['log_id'];
            $decoded['record'] = $row['record'];
            $logs[] = $decoded;
        }
        return $logs;
    }

    /**
     * Rehydrate a session by session_id using raw SQL (bypasses EM framework project scoping).
     * @param string $session_id
     * @param int $project_id
     * @return array Session object with 'messages', 'metadata', 'stats'
     */
    public function rehydrateProjectSession($session_id, $project_id, $username = null)
    {
        $pid = intval($project_id);
        $prefix = $this->PREFIX;

        // --- Preferred path: dedicated one-row-per-turn records ---
        // CappyChatTurn / CappyChatTurnError rows (written by logConversationTurn)
        // are already clean user-prompt + final-answer pairs, so rebuilding from
        // them is deterministic. Deriving turns from the per-LLM-call audit log
        // is lossy in agent mode (N rows/turn, injected TOOL RESULT messages and
        // tool-call-only envelopes leak in, errored turns vanish).
        $turnSql = "SELECT l.log_id, l.timestamp, l.record, l.message
                    FROM redcap_external_modules_log l
                    JOIN redcap_external_modules_log_parameters p
                        ON l.log_id = p.log_id AND p.name = 'session_id' AND p.value = ?
                    WHERE l.external_module_id = (
                        SELECT external_module_id FROM redcap_external_modules
                        WHERE directory_prefix = ? LIMIT 1
                    )
                    AND l.record IN ('CappyChatTurn', 'CappyChatTurnError')
                    ORDER BY l.log_id ASC";
        $turnResult = $this->query($turnSql, [$session_id, $prefix]);
        $turnLogs = [];
        $firstTurnLogId = null;
        $dbgRaw = 0; $dbgPidDropped = 0; $dbgUserDropped = 0; // PHI-safe counters
        while ($row = $turnResult->fetch_assoc()) {
            $dbgRaw++;
            $decoded = json_decode($row['message'], true);
            if ($decoded === null) continue;
            if (intval($decoded['project_id'] ?? 0) !== $pid) { $dbgPidDropped++; continue; }
            // Authorization: session ids are client-generated and guessable, so
            // scope to the requesting user. Turn rows carry the username that
            // created them; when a user is supplied, only that user's turns are
            // returned. Prevents rebuilding another user's chat (possible PHI)
            // from a guessed session_id within the same project.
            if ($username !== null && (string)($decoded['username'] ?? '') !== (string)$username) { $dbgUserDropped++; continue; }
            $decoded['timestamp'] = $row['timestamp'];
            $decoded['id'] = $row['log_id'];
            $decoded['record'] = $row['record'];
            if ($firstTurnLogId === null) $firstTurnLogId = (int)$row['log_id'];
            $turnLogs[] = $decoded;
        }

        // TEMP DIAGNOSTIC (PHI-safe: counts + ids only, never message content).
        // Remove once the rehydration-miss is pinpointed.
        $this->emDebug("rehydrateProjectSession", [
            'pid'                => $pid,
            'req_username'       => $username,        // REDCap login, not PHI
            'session_id'         => $session_id,      // client ms-timestamp, not PHI
            'turn_rows_raw'      => $dbgRaw,          // rows matched by session_id in SQL
            'dropped_pid'        => $dbgPidDropped,
            'dropped_username'   => $dbgUserDropped,
            'turn_rows_kept'     => count($turnLogs),
        ]);

        if (!empty($turnLogs)) {
            $session = $this->buildSessionFromTurns($session_id, $pid, $turnLogs);

            // Sessions active across the turn-logging deploy have their earlier
            // turns only as legacy audit rows. Merge those (rows before the first
            // turn row) so cross-deploy history isn't lost. Only when the request
            // is NOT user-scoped: legacy rows carry no username, so ownership is
            // unverifiable and they must not be exposed on a user-scoped request.
            if ($username === null && $firstTurnLogId !== null) {
                $legacy = $this->legacyReconstructSession($session_id, $pid, $prefix, $firstTurnLogId);
                if (!empty($legacy['messages'])) {
                    $session['messages'] = array_merge($legacy['messages'], $session['messages']);
                    $session['stats']['total_turns'] += ($legacy['stats']['total_turns'] ?? 0);
                    $session['stats']['total_tokens'] += ($legacy['stats']['total_tokens'] ?? 0);
                    $session['stats']['models_used'] = array_values(array_unique(array_merge(
                        $legacy['stats']['models_used'] ?? [],
                        $session['stats']['models_used'] ?? []
                    )));
                    if (!empty($legacy['metadata']['start_time'])) {
                        $session['metadata']['start_time'] = $legacy['metadata']['start_time'];
                    }
                }
            }
            return $session;
        }

        // No turn rows. A user-scoped request cannot be satisfied from legacy
        // audit rows — they carry no username, so ownership is unverifiable.
        // Deny rather than risk exposing another user's pre-deploy history.
        if ($username !== null) {
            return [
                'session_id' => $session_id,
                'messages' => [],
                'metadata' => [],
                'stats' => ['total_turns' => 0, 'total_tokens' => 0, 'models_used' => []]
            ];
        }

        // Unscoped fallback: reconstruct from the per-LLM-call audit log.
        return $this->legacyReconstructSession($session_id, $pid, $prefix);
    }

    /**
     * Legacy rehydration from the per-LLM-call audit log (SecureChatLog rows).
     * Used for sessions that predate turn logging, and to backfill the pre-turn
     * portion of sessions that straddle the deploy. $maxLogIdExclusive, when
     * given, limits the scan to rows strictly before that log_id.
     */
    private function legacyReconstructSession($session_id, $pid, $prefix, $maxLogIdExclusive = null)
    {
        $bound = $maxLogIdExclusive !== null ? " AND l.log_id < ?" : "";

        $sql = "SELECT l.log_id, l.timestamp, l.record, l.message
                FROM redcap_external_modules_log l
                JOIN redcap_external_modules_log_parameters p
                    ON l.log_id = p.log_id AND p.name = 'session_id' AND p.value = ?
                WHERE l.external_module_id = (
                    SELECT external_module_id FROM redcap_external_modules
                    WHERE directory_prefix = ? LIMIT 1
                )
                AND l.record IN ('SecureChatLog', 'SecureChatLogError')$bound
                ORDER BY l.log_id ASC";
        $params = $maxLogIdExclusive !== null
            ? [$session_id, $prefix, $maxLogIdExclusive]
            : [$session_id, $prefix];
        $result = $this->query($sql, $params);

        $logs = [];
        while ($row = $result->fetch_assoc()) {
            $decoded = json_decode($row['message'], true);
            if ($decoded === null) continue;
            if (intval($decoded['project_id'] ?? 0) !== $pid) continue;
            $decoded['timestamp'] = $row['timestamp'];
            $decoded['id'] = $row['log_id'];
            $decoded['record'] = $row['record'];
            $logs[] = $decoded;
        }

        // Fallback: scan JSON message for pre-promotion logs
        if (empty($logs)) {
            $bound2 = $maxLogIdExclusive !== null ? " AND log_id < ?" : "";
            $sql2 = "SELECT log_id, timestamp, record, message
                     FROM redcap_external_modules_log
                     WHERE external_module_id = (
                         SELECT external_module_id FROM redcap_external_modules
                         WHERE directory_prefix = ? LIMIT 1
                     )
                     AND record IN ('SecureChatLog', 'SecureChatLogError')
                     AND message LIKE ?$bound2
                     ORDER BY log_id ASC
                     LIMIT 1000";
            $pattern = '%"session_id":"' . preg_replace('/[^a-zA-Z0-9_-]/', '', $session_id) . '"%';
            $params2 = $maxLogIdExclusive !== null
                ? [$prefix, $pattern, $maxLogIdExclusive]
                : [$prefix, $pattern];
            $result2 = $this->query($sql2, $params2);
            while ($row = $result2->fetch_assoc()) {
                $decoded = json_decode($row['message'], true);
                if ($decoded === null) continue;
                if (intval($decoded['project_id'] ?? 0) !== $pid) continue;
                if (($decoded['session_id'] ?? '') !== $session_id) continue;
                $decoded['timestamp'] = $row['timestamp'];
                $decoded['id'] = $row['log_id'];
                $decoded['record'] = $row['record'];
                $logs[] = $decoded;
            }
        }

        if (empty($logs)) {
            return [
                'session_id' => $session_id,
                'messages' => [],
                'metadata' => [],
                'stats' => ['total_turns' => 0, 'total_tokens' => 0, 'models_used' => []]
            ];
        }

        $messages = [];
        $models = [];
        $total_tokens = 0;
        $first_timestamp = null;
        $last_timestamp = null;

        foreach ($logs as $index => $log) {
            if (!empty($log['error'])) continue;

            if (!empty($log['user_message'])) {
                $messages[] = [
                    'role' => 'user',
                    'content' => $log['user_message'],
                    'turn' => $index + 1
                ];
            }
            if (!empty($log['assistant_response'])) {
                $messages[] = [
                    'role' => 'assistant',
                    'content' => $log['assistant_response'],
                    'turn' => $index + 1,
                    'tools_used' => $log['tools_used'] ?? null
                ];
            }
            if (!empty($log['model'])) $models[$log['model']] = true;
            if (!empty($log['usage']['total_tokens'])) $total_tokens += $log['usage']['total_tokens'];
            if ($first_timestamp === null) $first_timestamp = $log['timestamp'];
            $last_timestamp = $log['timestamp'];
        }

        return [
            'session_id' => $session_id,
            'messages' => $messages,
            'metadata' => [
                'project_id' => $pid,
                'start_time' => $first_timestamp,
                'end_time' => $last_timestamp,
                'duration_seconds' => $first_timestamp && $last_timestamp ?
                    strtotime($last_timestamp) - strtotime($first_timestamp) : 0
            ],
            'stats' => [
                'total_turns' => count($messages) / 2,
                'total_tokens' => $total_tokens,
                'models_used' => array_keys($models)
            ]
        ];
    }

    private function filterDefaultParamsForModel($model, $params, ?string $modelId = null)
    {
        // Embedding models don't use chat defaultParams
        if ($model === 'ada-002' || $model === 'text-embedding-3-small' || $model === 'text-embedding-3-large') {
            $embedParams = array_merge([
                'model' => $modelId ?: $model
            ], $params);
            unset($embedParams['session_id'], $embedParams['agent_mode'], $embedParams['project_id'], $embedParams['log_turn'], $embedParams['token_policy']);
            return $embedParams;
        }

        $merged = array_merge($this->defaultParams, $params);

        // Only o1/o3-mini/gpt-5 get reasoning params
        if (!in_array($model, ['o1', 'o3-mini', 'gpt-5', 'gpt-5-6-sol', 'gpt-5-6-luna', 'gpt-5-6-terra'])) {
            unset($merged['reasoning']);
            unset($merged['reasoning_effort']);
        }

        // Only models supporting json_schema
        if (!in_array($model, self::SCHEMA_CAPABLE_MODELS, true)) {
            unset($merged['json_schema']);
        }

        // Only reasoning models have strict param set (use max_completion_tokens)
        if (in_array($model, ['o1', 'o3-mini', 'o3', 'o4-mini', 'gpt-5', 'gpt-5-6-sol', 'gpt-5-6-luna', 'gpt-5-6-terra'])) {
            $strict = [
                'model' => $model,
                'messages' => $merged['messages'] ?? [],
                'max_completion_tokens' => $merged['max_completion_tokens'] ?? ($merged['max_tokens'] ?? 32000),
            ];
            if (isset($merged['reasoning_effort'])) {
                $strict['reasoning_effort'] = $merged['reasoning_effort'];
            }

            // Preserve json_schema for reasoning models
            if (isset($merged['json_schema'])) {
                $strict['json_schema'] = $merged['json_schema'];
            }
            return $strict;
        }

        // Remove max_tokens for all non-o1/o3-mini
        unset($merged['max_tokens']);

        // Strip internal/meta params that are not part of the API contract
        unset($merged['session_id'], $merged['agent_mode'], $merged['project_id'], $merged['log_turn'], $merged['token_policy']);

        return $merged;
    }

    /**
     * Call a model.
     *
     * `$params['token_policy']` ('standard' | 'high', optional) picks which set of
     * AI Hub models — and which subscription key — serves the call. The call runs
     * under the stricter of: that value, the project's `token-policy-default`,
     * and the policy of any callAI() already executing on this instance. Asking
     * for less than that minimum, or for a model alias that is not configured
     * under the resolved policy, returns a TOKEN_POLICY_ERROR response; there is
     * no fallback to the other set of models. Omitted = the minimum, which is
     * 'standard' for a project with no default, so existing callers are unchanged.
     */
    public function callAI($model, $params = [], $project_id = null, $username = null)
    {
        $this->initSecureChatAI();
        $params = is_array($params) ? $params : [];

        try {
            $tokenPolicy = $this->resolveTokenPolicy($params['token_policy'] ?? null, $project_id);
        } catch (TokenPolicyException $e) {
            // Don't log a value we couldn't parse; the reason is in the message.
            unset($params['token_policy']);
            return $this->tokenPolicyError($model, $params, $project_id, $username, $e);
        }
        $params['token_policy'] = $tokenPolicy;

        $outerPolicy = $this->activeTokenPolicy;
        $this->activeTokenPolicy = $tokenPolicy;
        try {
            return $this->callAIUnderPolicy($model, $params, $project_id, $username);
        } finally {
            $this->activeTokenPolicy = $outerPolicy;
        }
    }

    /**
     * The minimum policy for a call: the project's default (if any) and the
     * policy of an enclosing callAI(), whichever is stricter.
     */
    private function minimumTokenPolicy(?int $project_id): string
    {
        $minimum = $this->activeTokenPolicy ?? TokenPolicy::STANDARD;
        if ($project_id) {
            $minimum = TokenPolicy::stricter($minimum, $this->getProjectMinimumTokenPolicy($project_id));
        }
        return $minimum;
    }

    /**
     * A project's Minimum Token Policy (`token-policy-default`), 'standard' when
     * unset. Used for the calling project and, by TokenPolicyPreHook, for any
     * project a tool reads.
     */
    public function getProjectMinimumTokenPolicy(int $project_id): string
    {
        try {
            return TokenPolicy::normalize($this->getProjectSetting('token-policy-default', $project_id))
                ?? TokenPolicy::STANDARD;
        } catch (TokenPolicyException $e) {
            // A stored value we don't recognize: fail closed.
            return TokenPolicy::HIGH;
        }
    }

    /**
     * @throws TokenPolicyException
     */
    private function resolveTokenPolicy($requested, $project_id): string
    {
        $project_id = $project_id ? (int)$project_id : null;
        return TokenPolicy::resolve(TokenPolicy::normalize($requested), $this->minimumTokenPolicy($project_id));
    }

    /**
     * The config for a model alias under a policy. Never falls back to the
     * other policy's models.
     *
     * @throws TokenPolicyException when the alias exists only under the other policy
     * @throws Exception 'Unsupported model' when no policy has it (unchanged behavior)
     */
    private function getModelConfigFor(string $model, string $tokenPolicy): array
    {
        if (isset($this->modelConfig[$tokenPolicy][$model])) {
            return $this->modelConfig[$tokenPolicy][$model];
        }
        $elsewhere = [];
        foreach ($this->modelConfig as $policy => $configs) {
            if ($policy !== $tokenPolicy && isset($configs[$model])) {
                $elsewhere[] = $policy;
            }
        }
        if (!$elsewhere) {
            throw new Exception('Unsupported model: ' . $model);
        }
        throw new TokenPolicyException(
            "Model '{$model}' is not configured under the '{$tokenPolicy}' token policy"
            . " (it exists only under: " . implode(', ', $elsewhere) . ")."
        );
    }

    /**
     * Error response for a call refused by the token policy. Not retried: the
     * same request would be refused again.
     */
    private function tokenPolicyError($model, array $params, $project_id, $username, TokenPolicyException $e): array
    {
        $error = [
            'error'   => true,
            'type'    => 'TOKEN_POLICY_ERROR',
            'message' => $e->getMessage(),
        ];
        // The message holds aliases and policy names only, never prompt content.
        $this->emDebug("Token policy refused call", [
            'model'   => $model,
            'pid'     => $project_id,
            'message' => $e->getMessage(),
        ]);
        if ($project_id) {
            $this->logErrorInteraction($project_id, $params + ['model' => $model], $error);
        }

        $sanitizedError = $this->sanitizeOutputForUI($error);
        $turnError = $sanitizedError;
        if (empty($turnError['model'])) {
            $turnError['model'] = $model;
        }
        $this->logConversationTurn($project_id, $params, $turnError, $username);

        return $sanitizedError;
    }

    private function callAIUnderPolicy($model, array $params, $project_id = null, $username = null)
    {
        $startTotal = microtime(true);
        $retries = 2;
        $attempt = 0;

        while ($attempt <= $retries) {
            try {
                $startAttempt = microtime(true);

                // --- Server-Side Compaction (opt-in, before agent mode gate) ---
                $params = $this->compactMessagesIfNeeded($params, $model);

                // --- Agent Mode Gate ---

                $agent_mode_requested = !empty($params['agent_mode']);
                $agent_mode_enabled = (bool) $this->getSystemSetting('enable_agent_mode');

                $response = null;

                if ($agent_mode_requested && $agent_mode_enabled) {
                    $startAgentLoop = microtime(true);
                    $response = $this->runAgentLoop(
                        model: $model,
                        params: $params,
                        project_id: $project_id,
                        username: $username
                    );
                    $this->emDebug("callAI timing - runAgentLoop", [
                        'duration_ms' => round((microtime(true) - $startAgentLoop) * 1000, 2)
                    ]);
                } else {
                    if($agent_mode_requested && !$agent_mode_enabled){
                        $this->emDebug("Agent mode requested but not enabled in system settings. Proceeding with normal LLM call.");
                        unset($params['agent_mode']);
                    }

                    // Normal single-call path
                    $startLLMCall = microtime(true);
                    $response = $this->callLLMOnce($model, $params, $project_id, $username);
                    $this->emDebug("callAI timing - callLLMOnce", [
                        'duration_ms' => round((microtime(true) - $startLLMCall) * 1000, 2)
                    ]);
                }

                // ✅ ALWAYS sanitize output before returning to UI
                $startSanitize = microtime(true);
                $sanitizedResponse = $this->sanitizeOutputForUI($response);
                $this->emDebug("callAI timing - sanitizeOutputForUI", [
                    'duration_ms' => round((microtime(true) - $startSanitize) * 1000, 2)
                ]);

                $this->emDebug("callAI timing - attempt complete", [
                    'attempt' => $attempt + 1,
                    'attempt_duration_ms' => round((microtime(true) - $startAttempt) * 1000, 2),
                    'total_duration_ms' => round((microtime(true) - $startTotal) * 1000, 2)
                ]);

                // One clean turn record for reliable rehydration. Log the
                // sanitized content (what the user actually saw), but carry over
                // tools_used/usage from the pre-sanitize response.
                $turnResponse = is_array($sanitizedResponse) ? $sanitizedResponse : ['content' => (string)$sanitizedResponse];
                if (empty($turnResponse['tools_used']) && !empty($response['tools_used'])) {
                    $turnResponse['tools_used'] = $response['tools_used'];
                }
                if (empty($turnResponse['usage']) && !empty($response['usage'])) {
                    $turnResponse['usage'] = $response['usage'];
                }
                if (!empty($response['error'])) {
                    $turnResponse['error'] = true;
                }
                $this->logConversationTurn($project_id, $params, $turnResponse, $username);

                return $sanitizedResponse;

            } catch (TokenPolicyException $e) {
                return $this->tokenPolicyError($model, $params, $project_id, $username, $e);
            } catch (\Exception $e) {
                $attempt++;
                // PHI-safe: log the error description and request metadata only.
                // Never log $params directly — it holds the prompt messages (PHI).
                $this->emDebug("Attempt $attempt failed", [
                    'model'         => $model,
                    'error'         => $e->getMessage(),
                    'message_count' => isset($params['messages']) && is_array($params['messages'])
                        ? count($params['messages'])
                        : 0,
                ]);

                if ($attempt > $retries) {
                    $error = [
                        'error' => true,
                        'type' => 'NETWORK_ERROR',
                        'message' => "Error after $retries retries: " . $e->getMessage()
                    ];
                    if ($project_id) {
                        $this->logErrorInteraction($project_id, $params, $error);
                    } else {
                        $this->emDebug("Skipping error logging due to missing project ID (pid).");
                    }

                    $this->emDebug("callAI timing - failed after retries", [
                        'total_duration_ms' => round((microtime(true) - $startTotal) * 1000, 2)
                    ]);

                    // ✅ Sanitize errors too
                    $sanitizedError = $this->sanitizeOutputForUI($error);

                    // Turn record so the user's question survives rehydration
                    // even when the request failed outright (no answer produced).
                    $turnError = is_array($sanitizedError) ? $sanitizedError : ['content' => (string)$sanitizedError];
                    $turnError['error'] = true;
                    if (empty($turnError['model'])) {
                        $turnError['model'] = $model;
                    }
                    $this->logConversationTurn($project_id, $params, $turnError, $username);

                    return $sanitizedError;
                }
            }
        }

        $this->emDebug("callAI timing - unknown error fallback", [
            'total_duration_ms' => round((microtime(true) - $startTotal) * 1000, 2)
        ]);

        // ✅ Sanitize fallback error
        return $this->sanitizeOutputForUI([
            'error' => true,
            'type' => 'UNKNOWN_ERROR',
            'message' => 'Unknown error'
        ]);
    }

    // -------------------------------------------------------------------------
    // Public inspection helpers (for test/diagnostic EMs)
    // -------------------------------------------------------------------------

    /**
     * Return the full tool catalog for a project (EM-discovered + JSON registry).
     */
    public function getToolCatalogForProject(?int $pid): array
    {
        return $this->loadToolsForProject($pid);
    }

    /**
     * Return the model aliases configured under a token policy.
     *
     * With no policy, it is the one a callAI() from $projectId (default: the
     * current project context) gets when it passes none — so a caller validating
     * its configured alias (MICA's assertModelIsRegistered) checks the right set
     * on a project whose Minimum Token Policy is `high`.
     */
    public function getAvailableModels(?string $tokenPolicy = null, ?int $projectId = null): array
    {
        $this->initSecureChatAI();
        try {
            $tokenPolicy = TokenPolicy::normalize($tokenPolicy);
        } catch (TokenPolicyException $e) {
            return [];
        }
        if ($tokenPolicy === null) {
            if ($projectId === null) {
                try {
                    $projectId = $this->getProjectId() ? (int) $this->getProjectId() : null;
                } catch (\Throwable $e) {
                    $projectId = null;
                }
            }
            $tokenPolicy = $this->minimumTokenPolicy($projectId);
        }
        return array_keys($this->modelConfig[$tokenPolicy] ?? []);
    }

    /**
     * Return the context window size (in tokens) for a model.
     */
    public function getModelContextWindow(string $model): int
    {
        return $this->getModelContextSpec($model)['context'] ?? 128000;
    }

    /**
     * Run server-side message compaction and return before/after stats.
     *
     * Makes one LLM call (to generate the summary) if compaction triggers.
     */
    public function runCompaction(array $messages, string $model, int $keepRecent = 6, ?string $summarizeModel = null, ?int $contextWindowOverride = null, ?string $tokenPolicy = null, ?int $projectId = null): array
    {
        $params = [
            'messages'             => $messages,
            'compact_if_needed'    => true,
            'compact_keep_recent'  => $keepRecent,
            'compact_model'        => $summarizeModel ?? $model,
            // Throws TokenPolicyException for a policy below the project minimum.
            'token_policy'         => $this->resolveTokenPolicy($tokenPolicy, $projectId),
        ];

        $beforeCount = count($messages);
        $beforeText  = $this->messagesToText($messages);
        $beforeTokens = $this->estimateTokens($beforeText, $model);

        $result = $this->compactMessagesIfNeeded($params, $model, $contextWindowOverride);

        $afterMessages = $result['messages'] ?? $messages;
        $afterCount    = count($afterMessages);
        $afterText     = $this->messagesToText($afterMessages);
        $afterTokens   = $this->estimateTokens($afterText, $model);

        $contextWindow = $contextWindowOverride ?? $this->getModelContextWindow($model);
        $threshold     = (int)($contextWindow * 0.8);

        return [
            'compacted'       => ($afterCount < $beforeCount),
            'context_window'  => $contextWindow,
            'threshold'       => $threshold,
            'before_messages' => $beforeCount,
            'before_tokens'   => $beforeTokens,
            'after_messages'  => $afterCount,
            'after_tokens'    => $afterTokens,
            'reduction_pct'   => $beforeTokens > 0
                ? round((1 - $afterTokens / $beforeTokens) * 100, 1)
                : 0,
            'messages'        => $afterMessages,
        ];
    }

    private function loadToolsForProject(?int $pid): array
    {
        if (empty($pid)) return [];

        // Discover tool definitions from enabled EMs (project-level overrides system-level)
        $emTools = $this->discoverEmToolDefinitions($pid);

        // Validate each tool definition
        $validatedTools = [];
        foreach ($emTools as $tool) {
            $validation = $this->validateToolDefinition($tool);
            if ($validation['valid']) {
                $validatedTools[] = $tool;
            } else {
                $this->emError("Invalid tool definition skipped", [
                    'tool_name' => $tool['name'] ?? 'unnamed',
                    'errors' => $validation['errors']
                ]);
            }
        }

        return $validatedTools;
    }

    /**
     * Discover tool definitions from enabled tool EMs.
     *
     * Reads agent-tool-definitions from config.json of EMs listed
     * in the agent_tool_em_prefixes setting.
     *
     * Project-level setting takes priority; falls back to system-level.
     *
     * @param int|null $pid Project ID for project-level prefix lookup
     * @return array Tool configs with auto-filled endpoint/module fields
     */
    private function discoverEmToolDefinitions(?int $pid = null): array
    {
        // Union of system-level (everyone gets) + project-level (this project only)
        $systemRaw  = $this->getSystemSetting('agent_tool_em_prefixes') ?? '';
        $projectRaw = (!empty($pid)) ? ($this->getProjectSetting('project_agent_tool_em_prefixes', $pid) ?? '') : '';

        $systemPrefixes  = $systemRaw  ? array_map('trim', explode(',', $systemRaw))  : [];
        $projectPrefixes = $projectRaw ? array_map('trim', explode(',', $projectRaw)) : [];
        $prefixes = array_values(array_unique(array_filter(array_merge($systemPrefixes, $projectPrefixes))));
        $tools = [];

        foreach ($prefixes as $prefix) {
            if (empty($prefix)) continue;

            try {
                $moduleDir = $this->getModuleDirectoryPath($prefix);

                // Read tool definitions from tools.json manifest
                $toolsPath = $moduleDir . '/tools.json';
                if (!file_exists($toolsPath)) {
                    $this->emDebug("tools.json not found for tool EM: {$prefix}");
                    continue;
                }

                $manifest = json_decode(file_get_contents($toolsPath), true);
                if (json_last_error() !== JSON_ERROR_NONE) {
                    $this->emError("Invalid tools.json for tool EM: {$prefix}");
                    continue;
                }

                $definitions = $manifest['tools'] ?? [];
                foreach ($definitions as $def) {
                    $action = $def['action'] ?? '';
                    $tools[] = [
                        'name'        => $def['name'] ?? '',
                        'description' => $def['description'] ?? '',
                        'endpoint'    => 'module_api',
                        'module'      => [
                            'prefix' => $prefix,
                            'action' => $action,
                        ],
                        'parameters'  => $def['parameters'] ?? ['type' => 'object', 'properties' => []],
                        'readOnly'    => $def['readOnly'] ?? false,
                        'destructive' => $def['destructive'] ?? false,
                    ];
                }

            } catch (\Throwable $e) {
                $this->emError("Failed to discover tools from {$prefix}: " . $e->getMessage());
            }
        }

        return $tools;
    }

    /**
     * Get the filesystem path for a REDCap external module by prefix.
     */
    private function getModuleDirectoryPath(string $prefix): string
    {
        if (method_exists('\ExternalModules\ExternalModules', 'getModuleDirectoryPath')) {
            $path = \ExternalModules\ExternalModules::getModuleDirectoryPath($prefix);
            if ($path !== false) {
                return $path;
            }
        }

        // Fallback: check both modules/ and modules-local/
        $docRoot = APP_PATH_DOCROOT . "../";
        foreach (['modules/', 'modules-local/'] as $dir) {
            $pattern = $docRoot . $dir . $prefix . "_v*";
            $matches = glob($pattern, GLOB_ONLYDIR);
            if (!empty($matches)) {
                return $matches[0];
            }
        }

        return $docRoot . "modules/{$prefix}";
    }

    /**
     * Validate tool definition structure
     * Returns ['valid' => bool, 'errors' => array]
     */
    private function validateToolDefinition(array $tool): array
    {
        $errors = [];

        // Required fields
        if (empty($tool['name'])) {
            $errors[] = "Missing 'name' field";
        } elseif (!preg_match('/^[a-zA-Z][a-zA-Z0-9_.]*$/', $tool['name'])) {
            $errors[] = "Invalid 'name' format (must start with letter, alphanumeric + _ . only)";
        }

        if (empty($tool['description'])) {
            $errors[] = "Missing 'description' field";
        }

        if (empty($tool['endpoint'])) {
            $errors[] = "Missing 'endpoint' field";
        } elseif (!in_array($tool['endpoint'], ['module_api', 'redcap_api', 'http'])) {
            $errors[] = "Invalid 'endpoint' value (must be: module_api, redcap_api, or http)";
        }

        // Validate parameters structure
        if (isset($tool['parameters'])) {
            if (!is_array($tool['parameters'])) {
                $errors[] = "'parameters' must be an object";
            } elseif (!isset($tool['parameters']['type']) || $tool['parameters']['type'] !== 'object') {
                $errors[] = "'parameters.type' must be 'object'";
            }
        }

        // Validate endpoint-specific requirements
        if ($tool['endpoint'] === 'redcap_api') {
            if (empty($tool['redcap']['prefix'])) {
                $errors[] = "Missing 'redcap.prefix' for redcap_api endpoint";
            }
            if (empty($tool['redcap']['action'])) {
                $errors[] = "Missing 'redcap.action' for redcap_api endpoint";
            }
        } elseif ($tool['endpoint'] === 'module_api') {
            if (empty($tool['module']['action'])) {
                $errors[] = "Missing 'module.action' for module_api endpoint";
            }
        }

        return [
            'valid' => empty($errors),
            'errors' => $errors
        ];
    }

    private function getAgentRouterPrompt(): string
    {
        $prompt = (string) ($this->getSystemSetting('agent_router_system_prompt') ?? '');
        if (!empty(trim($prompt))) return $prompt;

        // Safe default if not configured yet
        return implode("\n", [
            "You are the REDCap Agent Router.",
            "Your job:",
            "- Interpret user intent.",
            "- Decide if a tool is required.",
            "- Choose the correct tool(s) and provide a tool_call in strict JSON.",
            "- Ask for missing required parameters instead of guessing.",
            "- Never invent tool names or arguments.",
            "",
            "Output one of:",
            "1) A tool_call JSON object",
            "2) A clarification question",
            "3) A final natural-language answer"
        ]);
    }

    private function buildToolCatalogText(array $tools): string
    {
        if (empty($tools)) {
            return "TOOLS AVAILABLE: (none)";
        }

        $lines = ["TOOLS AVAILABLE:"];
        foreach ($tools as $t) {
            $name = $t['name'] ?? '(missing name)';
            $desc = $t['description'] ?? '';
            $required = $t['parameters']['required'] ?? [];
            $reqStr = !empty($required) ? implode(", ", $required) : "(none)";
            $lines[] = "- {$name}: {$desc} | required: {$reqStr}";
        }
        return implode("\n", $lines);
    }

    private function injectAgentSystemContext(array $messages, array $tools, int $depth = 0, int $maxDepth = 1): array
    {
        $routerPrompt = $this->getAgentRouterPrompt();

        // Add spawnAgent tool to catalog when depth allows
        if ($depth < $maxDepth) {
            $spawnTool = [
                'name'        => 'spawnAgent',
                'description' => 'Spawn a sub-agent to handle a task independently. '
                    . 'The sub-agent has its own tool pool and returns a final result.',
                'parameters'  => [
                    'type'       => 'object',
                    'properties' => [
                        'prompt' => [
                            'type'        => 'string',
                            'description' => 'Task for the sub-agent. Be specific and self-contained.'
                        ],
                        'allowed_tools' => [
                            'type'        => 'array',
                            'items'       => ['type' => 'string'],
                            'description' => 'Subset of tools the sub-agent can use. Empty = all tools.'
                        ]
                    ],
                    'required' => ['prompt']
                ]
            ];
            $tools[] = $spawnTool;
        }

        // ALWAYS include tool catalog in agent mode
        $routerPrompt .= "\n\n" . $this->buildToolCatalogText($tools);

        array_unshift($messages, [
            'role' => 'system',
            'content' => $routerPrompt
        ]);

        return $messages;
    }

    private function runAgentLoop(string $model, array $params, ?int $project_id, ?string $username = null, int $depth = 0, int $maxDepth = 1): array
    {
        $messages = $params['messages'] ?? [];
        $tools = $this->loadToolsForProject($project_id);

        // Force schema-capable model for agent mode. Pick the fallback
        // dynamically from the *configured* schema-capable models so we never
        // silently switch to an alias that isn't in $this->modelConfig (which
        // would throw "Unsupported model" mid-loop and surface as a generic
        // "network difficulties" error to the user).
        $tokenPolicy = $params['token_policy'] ?? $this->activeTokenPolicy ?? TokenPolicy::STANDARD;
        if (!in_array($model, self::SCHEMA_CAPABLE_MODELS, true)) {
            $configured = array_keys($this->modelConfig[$tokenPolicy] ?? []);
            $fallback = null;
            foreach (self::SCHEMA_CAPABLE_MODELS as $candidate) {
                if (in_array($candidate, $configured, true)) {
                    $fallback = $candidate;
                    break;
                }
            }
            if ($fallback === null) {
                throw new \Exception(
                    "Agent mode requires a schema-capable model but none of [" . implode(', ', self::SCHEMA_CAPABLE_MODELS)
                    . "] are configured in SecureChatAI's api-settings under the '{$tokenPolicy}' token policy. Add one, or disable agent mode for this project."
                );
            }
            $this->emDebug("Agent mode requires schema-capable model, switching from {$model} to {$fallback}");
            $model = $fallback;
        }

        $this->emDebug("AGENT MODE ENABLED", [
            'pid' => $project_id,
            'model' => $model,
            'tool_count' => count($tools),
            'tool_names' => array_column($tools, 'name')
        ]);

        // Initialize safety limits (reduced for sub-agents)
        $max_steps = (int) ($this->getSystemSetting('agent_max_steps') ?? 8);
        $max_tools = (int) ($this->getSystemSetting('agent_max_tools_per_run') ?? 15);
        $timeout = (int) ($this->getSystemSetting('agent_timeout_seconds') ?? 120);
        $max_tool_result_chars = (int) ($this->getSystemSetting('agent_max_tool_result_chars') ?? 8000);

        // Sub-agents get reduced limits to prevent runaway token usage
        if ($depth > 0) {
            $max_steps = max(2, (int) ($max_steps / 2));
            $max_tools = max(2, (int) ($max_tools / 2));
            $timeout = max(15, (int) ($timeout / 2));
        }
        $start_time = time();

        // CRITICAL: Agent mode needs higher token limit to avoid truncation mid-JSON
        // Reasoning models use max_completion_tokens, others use max_tokens
        if (in_array($model, ['o1', 'o3-mini', 'o3', 'o4-mini', 'gpt-5-6-sol', 'gpt-5-6-luna', 'gpt-5-6-terra'])) {
            $params['max_completion_tokens'] = 32000;
        } else {
            $params['max_tokens'] = 4000;
        }

        $step = 0;
        $tools_called = 0;
        $tool_call_history = []; // Track tool calls to detect loops
        // PHI-safe parallel to $tool_call_history for logging. The real signatures
        // embed json_encode($arguments), which can carry record data and filter
        // literals ([first_name]='Smith') — never loggable. These carry the SHAPE
        // only: tool name + sorted argument key names.
        $tool_call_shapes = [];
        $tools_used = []; // Track tools used for UI display

        // Inject router system prompt + tool catalog
        $messages = $this->injectAgentSystemContext($messages, $tools, $depth, $maxDepth);
        $params['messages'] = $messages;

        // ⚠️ IMPORTANT: disable native OpenAI tools for agent mode
        unset($params['tools'], $params['tool_choice']);

        // Prevent recursion
        unset($params['agent_mode']);

        // Add JSON schema for agent responses
        // Note: strict=false because tool arguments are dynamic (incompatible with strict mode)
        $params['json_schema'] = [
            'name' => 'agent_response',
            'strict' => false,
            'schema' => [
                'type' => 'object',
                'properties' => [
                    'tool_call' => [
                        'type' => 'object',
                        'properties' => [
                            'name' => ['type' => 'string'],
                            'arguments' => ['type' => 'object']
                        ],
                        'required' => ['name', 'arguments'],
                        'additionalProperties' => false
                    ],
                    'final_answer' => ['type' => 'string'],
                    'thinking' => ['type' => 'string']
                ],
                'additionalProperties' => false
            ]
        ];

        while ($step < $max_steps) {
            $step++;

            // Step-by-step logging (PHI-safe: log only metadata, not message content)
            $this->emDebug("AGENT STEP {$step}/{$max_steps}", [
                'messages_count' => count($messages),
                'last_message_length' => strlen(end($messages)['content'] ?? '')
            ]);

            $response = $this->callLLMOnce($model, $params, $project_id, $username);
            // PHI-safe: log only response metadata (token usage, model, shape) — never the
            // raw content body, which can contain prompt/response PHI.
            $this->emDebug("AGENT RAW RESPONSE (metadata only)", $this->summarizeResponseForDebug($response));

            $content = trim($response['content'] ?? '');

            // Decode HTML entities (REDCap may encode response)
            $content = html_entity_decode($content, ENT_QUOTES, 'UTF-8');

            // Clean ALL control characters (including literal newlines/tabs) that break JSON parsing
            // This collapses pretty-printed JSON into single line
            // Escaped sequences like \n are preserved since they're not actual control chars
            $cleanContent = preg_replace('/[\x00-\x1F]/', '', $content);

            $decoded = json_decode($cleanContent, true);

            // Fallback strategy for non-JSON responses
            if (json_last_error() !== JSON_ERROR_NONE) {
                $this->emDebug("JSON parse failed, trying regex extraction", [
                    'error' => json_last_error_msg(),
                    'content_preview' => substr($content, 0, 200)
                ]);

                // Try to extract tool_call JSON pattern from text
                if (preg_match('/\{[\s\S]*"tool_call"[\s\S]*\}/', $content, $matches)) {
                    $decoded = json_decode($matches[0], true);
                    if (json_last_error() === JSON_ERROR_NONE) {
                        $this->emDebug("Regex extraction succeeded");
                    }
                }

                // If still no valid JSON, treat as plain text final answer
                if (!is_array($decoded) || json_last_error() !== JSON_ERROR_NONE) {
                    $this->emDebug("Treating plain text response as final answer");
                    return [
                        'role' => 'assistant',
                        'content' => $content
                    ];
                }
            }

            if (isset($decoded['tool_call'])) {
                $tool_call = $decoded['tool_call'];
                $tool_name = $tool_call['name'] ?? null;
                $arguments = $tool_call['arguments'] ?? [];

                if (!$tool_name) {
                    return $this->agentError("INVALID_TOOL_CALL", "Tool name missing");
                }

                $execution = $this->executeToolCall(
                    tool_name: $tool_name,
                    arguments: $arguments,
                    tools: $tools,
                    project_id: $project_id,
                    model: $model,
                    username: $username
                );

                // Return errors immediately (but not MISSING_PARAMETERS - let agent handle that)
                if ($execution['error'] ?? false) {
                    return $execution;
                }

                // Cap tool result size to prevent token overflow
                $execution['result'] = $this->capToolResultSize(
                    $execution['result'],
                    $max_tool_result_chars
                );

                // Track tool usage for UI display. For records.search results
                // that carry a cache reference, also pass paging metadata
                // through so the chat UI can render its own pagination
                // controls against the server-side cache — the LLM shouldn't
                // burn turns clicking "next page".
                $usageEntry = [
                    'name' => $tool_name,
                    'arguments' => $arguments,
                    'step' => $step
                ];
                if ($tool_name === 'records.search' && is_array($execution['result'])) {
                    $r = $execution['result'];
                    if (!empty($r['reference']) && isset($r['total_record_count'])) {
                        $usageEntry['paging'] = [
                            'reference' => $r['reference'],
                            'total' => (int)$r['total_record_count'],
                            'offset' => (int)($r['offset'] ?? 0),
                            'limit' => (int)($r['limit'] ?? 20),
                            'truncated' => !empty($r['truncated']),
                            'preview_markdown' => $r['preview_markdown'] ?? '',
                        ];
                    }
                }
                $tools_used[] = $usageEntry;

                // Detect tool ping-pong loops (same tool+args called repeatedly)
                $callSignature = $tool_name . ':' . json_encode($arguments);
                $tool_call_history[] = $callSignature;
                $argKeys = is_array($arguments) ? array_keys($arguments) : [];
                sort($argKeys);
                // Flag the OUTCOME, not just the call. A tool that returns
                // {error:true,...} is the usual reason a model retries and loops, and
                // that error lives in the RESULT payload — $execution['error'] is
                // always false here, since line 1057 returns early on a hard failure.
                // Hook denials surface as result.status='denied' with error=false.
                $outcome = '';
                if (is_array($execution['result'] ?? null)) {
                    if (!empty($execution['result']['error'])) {
                        $outcome = ' [ERR]';
                    } elseif (($execution['result']['status'] ?? '') === 'denied') {
                        $outcome = ' [DENIED]';
                    }
                }
                $tool_call_shapes[] = $tool_name . '(' . implode(',', $argKeys) . ')' . $outcome;

                // Check for loops: same signature appearing 3+ times in last 5 calls
                if (count($tool_call_history) >= 5) {
                    $recentCalls = array_slice($tool_call_history, -5);
                    $signatureCounts = array_count_values($recentCalls);
                    if (max($signatureCounts) >= 3) {
                        // Log the whole 5-call window, not just a repeat count. Seeing
                        // the SEQUENCE is what makes these diagnosable — e.g. "call 1
                        // errored, so the model re-queried, then repeated the same
                        // paging call" reads very differently from "called X 3 times".
                        // PHI-safe: shapes only (see $tool_call_shapes above).
                        $this->emDebug("TOOL_LOOP_DETECTED — breaking agent loop", [
                            'step'          => $step,
                            'tools_called'  => $tools_called,
                            'elapsed_sec'   => time() - $start_time,
                            'repeat_count'  => max($signatureCounts),
                            'recent_shapes' => array_slice($tool_call_shapes, -5),
                        ]);
                        return $this->agentError(
                            "TOOL_LOOP_DETECTED",
                            "Agent is repeatedly calling the same tool. Breaking loop to prevent infinite execution."
                        );
                    }
                }

                // Increment tool counter and check limit
                $tools_called++;
                if ($tools_called >= $max_tools) {
                    return $this->agentError(
                        "MAX_TOOLS_EXCEEDED",
                        "Agent called {$tools_called} tools (limit: {$max_tools})"
                    );
                }

                // Check timeout
                if (time() - $start_time > $timeout) {
                    return $this->agentError(
                        "TIMEOUT",
                        "Agent exceeded {$timeout} second time limit"
                    );
                }

                // ✅ Inject tool result as USER context (standard practice)
                $messages[] = [
                    'role' => 'user',
                    'content' =>
                        "TOOL RESULT [{$tool_name}]:\n" .
                        json_encode($execution['result'], JSON_PRETTY_PRINT)
                ];

                $params['messages'] = $messages;
                continue;
            } elseif (isset($decoded['final_answer'])) {
                return [
                    'role' => 'assistant',
                    'content' => $decoded['final_answer'],
                    'tools_used' => $tools_used // Include tool metadata for UI
                ];
            } else {
                // No tool_call or final_answer - treat entire response as final answer
                $this->emDebug("No tool_call or final_answer field, using raw content");
                return [
                    'role' => 'assistant',
                    'content' => $content,
                    'tools_used' => $tools_used
                ];
            }
        }

        return $this->agentError(
            "AGENT_MAX_STEPS_EXCEEDED",
            "Agent exceeded maximum allowed steps ({$max_steps})"
        );
    }

    private function callLLMOnce(string $model, array $params, ?int $project_id, ?string $username = null): array
    {
        $this->initSecureChatAI();

        // Every internal path (agent loop, sub-agents, compaction) lands here, so
        // this is where the policy picks the model set. It never reaches the
        // provider: unknown params can 400 on Azure/Bedrock.
        $tokenPolicy = TokenPolicy::resolve(
            TokenPolicy::normalize($params['token_policy'] ?? null),
            $this->activeTokenPolicy ?? TokenPolicy::STANDARD
        );
        unset($params['token_policy']);

        $modelConfig = $this->getModelConfigFor($model, $tokenPolicy);
        $api_endpoint = $modelConfig['api_url'];

        foreach ($modelConfig['required'] as $param) {
            if (empty($params[$param])) {
                throw new Exception('Missing required parameter: ' . $param);
            }
        }

        $filteredParams = $this->filterDefaultParamsForModel($model, $params, $modelConfig['model_id']);

        [$paramName, $dynamicMax, $promptTokens] = $this->computeDynamicMaxTokens($model, $fullPrompt ?? $filteredParams['messages'][0]['content'] ?? '');
        $filteredParams[$paramName] = (int)$dynamicMax;
        $this->emDebug("Dynamic tokens: prompt={$promptTokens}, max={$dynamicMax} for {$model}");

        // Route to appropriate adapter based on model alias
        if (in_array($model, ['ada-002', 'text-embedding-3-small', 'text-embedding-3-large']) || str_contains($model, 'embed')) {
            // Embeddings
            $gpt = new GPTModelRequest($this, $modelConfig, $this->defaultParams, $model);
            $responseData = $gpt->sendRequest($api_endpoint, $filteredParams);

        } elseif (str_contains($model, 'whisper')) {
            // Speech-to-text
            $whisper = new WhisperModelRequest($this, $modelConfig, $this->defaultParams, $model);
            $responseData = $whisper->sendRequest($api_endpoint, $params);

        } elseif (str_contains($model, 'tts')) {
            // Text-to-speech
            $tts = new GPT4oMiniTTSModelRequest($this, $modelConfig, $this->defaultParams, $model);
            $responseData = $tts->sendRequest($api_endpoint, $params);

        } elseif (str_starts_with($model, 'claude') || str_contains($model, 'anthropic')) {
            // Claude (Bedrock format)
            $claude = new ClaudeModelRequest($this, $modelConfig, $this->defaultParams, $model);
            $responseData = $claude->sendRequest($api_endpoint, $filteredParams);

        } else {
            // OpenAI-compatible chat models (Azure AI Foundry, Vertex AI, Llama, etc.)
            $filteredParams = $this->filterDefaultParamsForModel($model, $params, $modelConfig['model_id']);
            $generic = new GenericModelRequest($this, $modelConfig, [], $model);
            $responseData = $generic->sendRequest($api_endpoint, $filteredParams);
            // PHI-safe: log only response metadata (token usage, model, shape) — never the
            // raw API response body, which can contain response PHI.
            $this->emDebug("RAW GenericModelRequest API RESPONSE (metadata only)", $this->summarizeResponseForDebug($responseData));
        }

        $normalizedResponse = $this->normalizeResponse($responseData, $model);

        if ($project_id) {
            $this->logInteraction($project_id, $params + ['token_policy' => $tokenPolicy], $responseData, $username);
        }

        return $normalizedResponse;
    }

    private function executeToolCall(
        string $tool_name,
        array $arguments,
        array $tools,
        ?int $project_id,
        string $model = 'o3-mini',
        ?string $username = null
    ): array {
        // Built-in: spawnAgent (before pipeline lookup)
        if ($tool_name === 'spawnAgent') {
            return $this->handleSpawnAgentCall($arguments, $project_id, $model, $username);
        }

        // Build tool use + context
        $toolUse = new ToolUse($tool_name, $arguments);
        $context = new ToolContext($project_id, $username);
        $context->set('secure_chat_ai_instance', $this);
        $context->set('guzzle_client', $this->guzzleClient);
        $context->set('model', $model);
        // For TokenPolicyPreHook, and for tool EMs that make their own LLM call in a
        // separate request (activeTokenPolicy doesn't reach a new PHP request).
        $context->set('token_policy', $this->activeTokenPolicy ?? TokenPolicy::STANDARD);

        // Phase 1: Lookup tool config from JSON registry
        $toolConfig = null;
        foreach ($tools as $t) {
            if (($t['name'] ?? null) === $tool_name) {
                $toolConfig = $t;
                break;
            }
        }

        if (!$toolConfig) {
            // NON-FATAL (changed 2026-08-24). This used to return agentError(), which sets
            // error=true and makes runAgentLoop() abort the whole turn — the user lost the
            // answer to everything else they asked and got a generic "I don't have access to
            // that capability" instead. A model asking for a tool that isn't registered is a
            // recoverable mistake, not a system failure, so feed it back the same way
            // MISSING_PARAMETERS and hook denials are handled below and let the agent adapt
            // in-turn. Matters whenever a tool exists in an EM's router but not its
            // tools.json manifest (records.aggregate, projects.search), or was deliberately
            // withdrawn (records.save, page.fill — write-disabled 2026-08-24).
            // Repeat calls are bounded by tool-loop detection plus max_steps/max_tools.
            $this->emDebug("UNKNOWN_TOOL (non-fatal)", ['tool' => $tool_name]);
            return [
                'error'  => false,
                'type'   => 'UNKNOWN_TOOL',
                'result' => [
                    'status'  => 'unavailable',
                    'message' => "Tool '{$tool_name}' is not available and does not exist. "
                        . "Do not call it again. Use only the tools you were given; if none of "
                        . "them can do what the user asked, say so plainly and stop.",
                ],
            ];
        }

        // Wrap config as typed tool
        $toolAdapter = new JsonConfigToolAdapter($toolConfig);

        // Build pipeline with hooks from config (system + project)
        $pipeline = new ToolPipeline(
            resolver: fn() => $toolAdapter,
            preHooks: $this->loadPreToolUseHooks($project_id),
            postHooks: $this->loadPostToolUseHooks($project_id)
        );

        // Run pipeline (Phases 2-7: parse, validate, pre-hooks, permissions, execute, post-hooks)
        $result = $pipeline->handle($toolUse, $context);

        // Preserve MISSING_PARAMETERS behavior: return error=false so agent can ask naturally
        if ($result->isError && str_contains($result->errorMessage ?? '', 'Missing required')) {
            $missing = str_replace('Missing required parameters: ', '', $result->errorMessage);
            return [
                'error'   => false,
                'type'    => 'MISSING_PARAMETERS',
                'missing' => array_map('trim', explode(',', $missing)),
                'result'  => [
                    'status'         => 'incomplete',
                    'message'        => $result->errorMessage,
                    'missing_fields' => array_map('trim', explode(',', $missing))
                ]
            ];
        }

        // Hook denials should be fed back to the agent (not treated as fatal errors)
        // so the agent can explain what happened to the user
        if ($result->isError && (
            str_contains($result->errorMessage ?? '', 'denied by hook') ||
            str_contains($result->errorMessage ?? '', 'Blocked by')
        )) {
            // Denials were previously invisible in the EM log — hooks report via
            // error_log(), which lands in php_errors.log with no request id, so a
            // denied turn could not be correlated with the rest of the trace. Log it
            // here, where we still have the request context.
            // PHI-safe: tool name plus a developer-authored hook message.
            $this->emDebug("TOOL DENIED BY HOOK", [
                'tool'    => $tool_name,
                'message' => $result->errorMessage,
            ]);
            return [
                'error'  => false,
                'result' => [
                    'status'  => 'denied',
                    // A denial is TERMINAL — a hook is deterministic, so the identical
                    // call denies identically. Saying so is what stops the retry loop:
                    // page.highlight was denied 5x in a row on PID 70 and the model
                    // kept re-issuing the same call until the loop detector fired.
                    'message' => $result->errorMessage
                        . ' This decision is final and will not change on retry — do NOT'
                        . ' call this tool again with these arguments. Tell the user plainly'
                        . ' what you could not do, and continue with what you can.',
                ]
            ];
        }

        return $result->toArray();
    }

    /**
     * Load PreToolUse hooks from system + project settings.
     *
     * @return PreToolUseHook[]
     */
    private function loadPreToolUseHooks(?int $projectId = null): array
    {
        // ProjectAccessPreHook and TokenPolicyPreHook are always-on — core
        // governance, not configurable
        $builtin = [new ProjectAccessPreHook(), new TokenPolicyPreHook($this)];
        $system  = $this->loadHooksFromSetting('pre_tool_use_hooks', PreToolUseHook::class, null);
        $project = $projectId
            ? $this->loadHooksFromSetting('project_pre_tool_use_hooks', PreToolUseHook::class, $projectId)
            : [];
        $hooks = array_merge($builtin, $system, $project);

        // Project-level opt-out: the PHI field protection hook is ON by default,
        // but a project may disable it (all current models accept PHI).
        if ($projectId && $this->getProjectSetting('disable_phi_field_hook', $projectId)) {
            $hooks = array_values(array_filter($hooks, fn($hook) => !$this->isPhiFieldHook($hook)));
            $this->emDebug("PHI field pre-hook disabled via project setting for project {$projectId}");
        }

        return $hooks;
    }

    /**
     * True if the given hook instance is the PHI field protection hook
     * (matched by class basename so it works regardless of installed EM version).
     */
    private function isPhiFieldHook($hook): bool
    {
        $class = get_class($hook);
        $needle = '\\PhiFieldPreHook';
        return $class === 'Stanford\\REDCapAgentRecordTools\\PhiFieldPreHook'
            || substr($class, -strlen($needle)) === $needle;
    }

    /**
     * Load PostToolUse hooks from system + project settings.
     *
     * @return PostToolUseHook[]
     */
    private function loadPostToolUseHooks(?int $projectId = null): array
    {
        $system  = $this->loadHooksFromSetting('post_tool_use_hooks', PostToolUseHook::class, null);
        $project = $projectId
            ? $this->loadHooksFromSetting('project_post_tool_use_hooks', PostToolUseHook::class, $projectId)
            : [];
        return array_merge($system, $project);
    }

    /**
     * Load and instantiate hook classes from a comma-separated setting.
     *
     * @param string $settingKey  Setting name
     * @param string $interfaceName  Required interface
     * @param int|null $projectId  If provided, reads project setting; otherwise system setting
     * @return array
     */
    private function loadHooksFromSetting(string $settingKey, string $interfaceName, ?int $projectId = null): array
    {
        $raw = $projectId !== null
            ? $this->getProjectSetting($settingKey, $projectId)
            : $this->getSystemSetting($settingKey);

        if (empty($raw)) {
            return [];
        }

        $classNames = array_map('trim', explode(',', $raw));
        $hooks = [];

        foreach ($classNames as $className) {
            if (empty($className) || !class_exists($className)) {
                $this->emDebug("Hook class not found: {$className}");
                continue;
            }

            $instance = new $className();
            if ($instance instanceof $interfaceName) {
                $hooks[] = $instance;
            } else {
                $this->emDebug("Hook class {$className} does not implement {$interfaceName}");
            }
        }

        return $hooks;
    }

    private function askForMoreInfoText(string $tool_name, array $missing_fields): string
    {
        return "I need: " . implode(", ", $missing_fields) . " to use {$tool_name}.";
    }

    private function agentError(string $type, string $message): array
    {
        // Log EVERY agent abort here rather than at the ~10 call sites. The user only
        // ever sees a deliberately vague apology (see the AGENT_ERROR_MESSAGES map),
        // so without this the log showed nothing but AGENT STEP lines stopping dead —
        // there was no way to answer "why did Cappy give up?" after the fact.
        //
        // PHI-safe: $type is a fixed constant and $message is a developer-authored
        // diagnostic. Neither carries record data. Callers that have richer context
        // (the loop detector, the tool-limit checks) log it themselves just before
        // calling, so this stays the backstop rather than the whole story.
        //
        // Sub-agent aborts land here too and look identical to top-level ones; the
        // $type is enough to tell them apart in practice (MAX_DEPTH_EXCEEDED,
        // SUBAGENTS_DISABLED and MISSING_PROMPT only come from the spawn path).
        $this->emDebug("AGENT ABORT [{$type}]", ['message' => $message]);

        return [
            'error' => true,
            'type' => $type,
            'message' => $message
        ];
    }

    /**
     * Handle the built-in spawnAgent tool call.
     */
    private function handleSpawnAgentCall(array $arguments, ?int $project_id, string $model, ?string $username = null): array
    {
        $prompt = $arguments['prompt'] ?? '';
        if (empty($prompt)) {
            return $this->agentError("MISSING_PROMPT", "spawnAgent requires a 'prompt' argument");
        }

        $maxDepth = (int) ($this->getSystemSetting('agent_max_subagent_depth') ?? 1);
        if ($maxDepth <= 0) {
            return $this->agentError("SUBAGENTS_DISABLED", "Sub-agents are disabled (agent_max_subagent_depth=0)");
        }

        $allowedTools = $arguments['allowed_tools'] ?? [];

        return $this->spawnAgent(
            prompt: $prompt,
            model: $model,
            projectId: $project_id,
            username: $username,
            depth: 1,
            maxDepth: $maxDepth,
            allowedTools: $allowedTools
        );
    }

    /**
     * Spawn a sub-agent to handle a task independently.
     *
     * The sub-agent runs a fresh agent loop with reduced limits
     * and returns its final answer as a tool result.
     */
    private function spawnAgent(
        string $prompt,
        string $model,
        ?int $projectId,
        ?string $username = null,
        int $depth = 1,
        int $maxDepth = 1,
        array $allowedTools = []
    ): array
    {
        $this->emDebug("SPAWN AGENT (depth={$depth}/{$maxDepth})", [
            'prompt_preview' => substr($prompt, 0, 100),
            'model' => $model,
            'allowed_tools' => $allowedTools ?: '(all)'
        ]);

        // Depth guard
        if ($depth > $maxDepth) {
            return $this->agentError(
                "MAX_DEPTH_EXCEEDED",
                "Sub-agent depth {$depth} exceeds max {$maxDepth}"
            );
        }

        // Load tools for the project
        $allTools = $this->loadToolsForProject($projectId);

        // Scope tools if requested
        if (!empty($allowedTools)) {
            $allTools = array_values(array_filter($allTools, function ($t) use ($allowedTools) {
                return in_array($t['name'] ?? '', $allowedTools);
            }));
        }

        // Build fresh messages from the prompt
        $messages = [
            ['role' => 'user', 'content' => $prompt]
        ];

        // Reduced limits for sub-agents. A sub-agent runs under its parent's
        // token policy; it only ever runs inside the parent's callAI().
        $params = [
            'messages' => $messages,
            'max_tokens' => 4000,
            'token_policy' => $this->activeTokenPolicy ?? TokenPolicy::STANDARD,
        ];

        // Run child agent loop
        $result = $this->runAgentLoop(
            model: $model,
            params: $params,
            project_id: $projectId,
            username: $username,
            depth: $depth,
            maxDepth: $maxDepth
        );

        // Format result for parent
        if (!empty($result['error'])) {
            return [
                'error'  => false, // Don't propagate as hard error to parent
                'result' => [
                    'status' => 'failed',
                    'error'  => $result['message'] ?? 'Sub-agent failed',
                ]
            ];
        }

        $finalAnswer = $result['content'] ?? $result['message'] ?? '(no result)';

        return [
            'error'  => false,
            'result' => [
                'status' => 'completed',
                'result' => $finalAnswer,
            ]
        ];
    }

    /**
     * Cap tool result size to prevent token overflow
     * Intelligently truncates while preserving structure and adding metadata
     */
    private function capToolResultSize($result, int $maxChars)
    {
        $jsonResult = json_encode($result, JSON_UNESCAPED_UNICODE);
        $originalSize = strlen($jsonResult);

        // If under limit, return as-is
        if ($originalSize <= $maxChars) {
            return $result;
        }

        $truncated = $result;
        $wasTruncated = false;
        // Populated by the object branch; reported to both the model and the log.
        $droppedKeys = [];

        // Handle arrays - truncate items
        if (is_array($result) && array_is_list($result)) {
            $itemCount = count($result);
            $kept = [];
            $currentSize = 2; // [] brackets

            foreach ($result as $item) {
                $itemJson = json_encode($item, JSON_UNESCAPED_UNICODE);
                $itemSize = strlen($itemJson) + 1; // +1 for comma

                if ($currentSize + $itemSize > $maxChars - 200) { // Reserve 200 chars for metadata
                    $wasTruncated = true;
                    break;
                }

                $kept[] = $item;
                $currentSize += $itemSize;
            }

            $truncated = $kept;
            if ($wasTruncated) {
                $truncated[] = [
                    '_truncated' => true,
                    '_original_count' => $itemCount,
                    '_returned_count' => count($kept),
                    '_message' => "Result truncated to fit token budget. Showing " . count($kept) . " of $itemCount items."
                ];
            }
        }
        // Handle objects - recursively cap nested values
        elseif (is_array($result)) {
            $currentSize = 2; // {} brackets
            $truncated = [];

            foreach ($result as $key => $value) {
                $valueJson = json_encode($value, JSON_UNESCAPED_UNICODE);
                $itemSize = strlen($key) + strlen($valueJson) + 4; // key + value + quotes/colon

                // Reserve room for the truncation metadata below. It used to be a
                // one-line message that fit in 200; the explanatory version plus
                // _dropped_keys needs more, and overshooting $maxChars would defeat
                // the point of the cap. Clamped to a quarter of the budget so a
                // small $maxChars can't push the threshold negative — that would
                // break on the FIRST key and return metadata with no data at all.
                if ($currentSize + $itemSize > $maxChars - min(700, intdiv($maxChars, 4))) {
                    $wasTruncated = true;
                    break;
                }

                $truncated[$key] = $value;
                $currentSize += $itemSize;
            }

            if ($wasTruncated) {
                // Name the dropped keys FOR THE MODEL, not just for our log. The old
                // message said only "truncated to fit token budget", so a model that
                // asked for a specific key (records.search + include_records=true)
                // received a response with that key silently absent, no indication it
                // had ever existed, and a tool `note` telling it to ask again — so it
                // did, until the loop detector killed the turn. Telling it what is
                // gone and that retrying won't help is what breaks that cycle.
                $droppedKeys = array_values(array_diff(array_keys($result), array_keys($truncated)));
                $truncated['_truncated'] = true;
                // NOTE: assigning the meta keys below does not disturb $droppedKeys —
                // array_diff only subtracts, and these keys aren't in $result.
                $truncated['_dropped_keys'] = $droppedKeys;
                // The remedy has to be one that CAN work. The previous version
                // said "do not retry" and then "request fewer records via
                // limit" — and the model, reasonably, did the actionable half.
                // Observed in prod 2026-08-24: limit 500 -> 100 -> 100 -> 200
                // -> 100 across 8 steps before the step cap killed the turn.
                // Lowering the record count cannot help when the overflow is
                // driven by per-record WIDTH (a 141-option checkbox is ~1.3KB
                // per record, so ~6 records fit in 8000 chars whatever the
                // limit says). Narrowing FIELDS is the remedy that works.
                $overflow = $maxChars > 0 ? round($originalSize / $maxChars, 1) : 0;
                $truncated['_message'] = "Result truncated to fit token budget. "
                    . "These keys were REMOVED and are NOT available: " . implode(', ', $droppedKeys) . ". "
                    . "The full result was {$originalSize} chars against a budget of {$maxChars} "
                    . "({$overflow}x over). Re-requesting the same keys returns the same truncation — "
                    . "do NOT retry, and do NOT retry with a smaller record limit: when a result is "
                    . "this far over budget the cause is usually how WIDE each record is, not how many "
                    . "there are, so a smaller limit truncates identically. What does work: answer from "
                    . "the keys you did receive; or ask for fewer FIELDS; or ask a question that returns "
                    . "a single value instead of rows. If you cannot answer from what you received, say "
                    . "so plainly and tell the user what you would need — do not keep calling tools.";
            }
        }
        // Handle strings - simple truncation
        elseif (is_string($result)) {
            $truncated = substr($result, 0, $maxChars - 100);
            $truncated .= "\n\n[... truncated " . ($originalSize - strlen($truncated)) . " characters to fit token budget]";
        }

        // Never truncate silently. Reaching this line means we're over the cap (the
        // under-limit case returned early), so log unconditionally — that covers all
        // three branches above and any added later.
        //
        // This blind spot hid a real bug: records.search puts its (potentially huge)
        // `records` payload in the LAST key, the object branch above stops at the first
        // key that doesn't fit, so a model that explicitly passed include_records=true
        // got a response with `records` quietly gone — and looped asking for it again.
        // The dropped key names are the whole diagnostic.
        //
        // PHI-safe: key NAMES and byte counts only, never values. $droppedKeys is
        // set by the object branch (the only shape where whole keys disappear); it
        // stays empty for list/string truncation, where nothing is keyed.
        $this->emDebug("TOOL RESULT TRUNCATED", [
            'shape'          => is_array($result)
                ? (array_is_list($result) ? 'list' : 'object')
                : gettype($result),
            'original_chars' => $originalSize,
            'max_chars'      => $maxChars,
            'dropped_keys'   => $droppedKeys,
        ]);

        return $truncated;
    }

    /**
     * Final cleanup: ensure response is always clean, user-ready text
     * - Extract text from stringified JSON (agent schema or accidental)
     * - Convert error objects to polite messages
     * - Never return raw JSON strings or error arrays to the UI
     */
    private function sanitizeOutputForUI(array $response): array
    {
        // Embedding responses (ada-002) should pass through unchanged
        // They have 'data' field with embedding vectors, not 'content'
        if (isset($response['data']) && !isset($response['content'])) {
            $this->emDebug("Passing through embedding response in sanitizeOutputForUI");
            return $response;
        }

        // TTS responses should pass through unchanged (have audio_base64)
        if (isset($response['audio_base64'])) {
            $this->emDebug("Passing through TTS response in sanitizeOutputForUI");
            return $response;
        }

        // STT/Whisper responses should pass through unchanged (have 'text' from transcription)
        // Raw whisper responses don't have 'role' field
        if (isset($response['text']) && !isset($response['role'])) {
            $this->emDebug("Passing through STT response in sanitizeOutputForUI");
            return $response;
        }

        // Handle error responses with friendly messages
        if (!empty($response['error'])) {
            $type = $response['type'] ?? 'UNKNOWN_ERROR';
            $msg = $response['message'] ?? 'An error occurred';

            // A token-policy refusal is a caller/config error the calling EM must
            // be able to detect, so it keeps its error flag, type and reason.
            if ($type === 'TOKEN_POLICY_ERROR') {
                return [
                    'role'    => 'assistant',
                    'content' => "This request was not sent: it is not allowed under the AI token policy. Please contact your administrator.",
                    'error'   => true,
                    'type'    => $type,
                    'message' => $msg,
                ];
            }

            $friendlyMessages = [
                'TIMEOUT' => "I apologize, but that request took longer than expected. Please try again in a moment.",
                'MAX_TOOLS_EXCEEDED' => "I apologize, but I needed to use too many tools for that request. Could you try simplifying your question?",
                'AGENT_MAX_STEPS_EXCEEDED' => "I apologize, but I couldn't complete that task efficiently. Could you try breaking it into smaller requests?",
                'TOOL_LOOP_DETECTED' => "I apologize, but I got stuck in a loop trying to complete that request. Could you try rephrasing your question?",
                'UNKNOWN_TOOL' => "I apologize, but I don't have access to that capability right now.",
                'MISCONFIGURED_AGENT_TOOLS' => "I apologize, but I'm experiencing technical difficulties. Please contact your administrator.",
                'REDCAP_API_ERROR' => "I apologize, but I'm having trouble accessing that data right now. Please try again in a moment.",
                'NETWORK_ERROR' => "I apologize, but I'm experiencing network difficulties. Please wait a moment and try again.",
            ];

            $politeMessage = $friendlyMessages[$type] ??
                "I apologize, but I'm experiencing technical difficulties. Please wait a moment and try again.";

            return [
                'role' => 'assistant',
                'content' => $politeMessage
            ];
        }

        $content = trim($response['content'] ?? '');

        // Decode HTML entities (in case of double-encoding)
        $content = html_entity_decode($content, ENT_QUOTES, 'UTF-8');

        // Try to detect and parse JSON responses
        if (!empty($content) && ($content[0] === '{' || str_starts_with($content, '```json'))) {
            // Strip markdown code fences if present
            $content = preg_replace('/^```json\s*|\s*```$/s', '', $content);
            $content = trim($content);

            // Clean control characters that break JSON parsing (same as agent loop)
            $cleanContent = preg_replace('/[\x00-\x1F]/', '', $content);

            $decoded = json_decode($cleanContent, true);
            $parseError = json_last_error();

            if ($parseError === JSON_ERROR_NONE && is_array($decoded)) {
                if (!empty($response['preserve_structure'])) {
                    // Keep the JSON string as-is for structured output
                    $response['content'] = $content;
                    return $response;
                }

                // Extract from agent schema format
                if (isset($decoded['final_answer'])) {
                    $content = $decoded['final_answer'];
                } elseif (isset($decoded['tool_call'])) {
                    // If we're seeing a tool_call in final output, something went wrong
                    $content = "I apologize, but I wasn't able to complete that request properly. Could you try rephrasing?";
                } elseif (isset($decoded['content'])) {
                    // Some models nest content in JSON
                    $content = $decoded['content'];
                } elseif (isset($decoded['message'])) {
                    $content = $decoded['message'];
                }
                // Otherwise leave the JSON string as-is (might be intentional structured output)
            }
        }

        // EMERGENCY BACKSTOP: If content STILL looks like our JSON schema, strip it one more time
        // This handles edge cases where json_decode failed or we didn't extract properly
        // Works even with truncated JSON (missing closing brace/quote)
        if (!empty($content) && preg_match('/\{\s*"final_answer"\s*:\s*"(.*)$/s', $content, $match)) {
            $extracted = $match[1];
            // Remove trailing garbage (incomplete JSON structure)
            $extracted = preg_replace('/["}\s]*$/', '', $extracted);
            // Unescape JSON string escapes
            $content = str_replace(['\n', '\r', '\t', '\"', '\\\\'], ["\n", "\r", "\t", '"', '\\'], $extracted);
        }

        $sanitized = [
            'role' => $response['role'] ?? 'assistant',
            'content' => $content,
            'model' => $response['model'] ?? null,
            'usage' => $response['usage'] ?? null
        ];

        // Preserve tool metadata if present (for UI display)
        if (!empty($response['tools_used'])) {
            $sanitized['tools_used'] = $response['tools_used'];
        }

        return $sanitized;
    }

    /**
     * Convert a messages array to a readable string for token estimation
     * and summarization prompts.
     */
    private function messagesToText(array $messages): string
    {
        $lines = [];
        foreach ($messages as $msg) {
            $role = $msg['role'] ?? 'unknown';
            $content = $msg['content'] ?? '';

            // Handle content arrays (some models use [{type: "text", text: "..."}])
            if (is_array($content)) {
                $parts = [];
                foreach ($content as $block) {
                    if (isset($block['text'])) $parts[] = $block['text'];
                }
                $content = implode("\n", $parts);
            }

            $lines[] = "[{$role}]: {$content}";
        }
        return implode("\n", $lines);
    }

    /**
     * Compact messages if they exceed the model's context window.
     *
     * Opt-in via params:
     *   compact_if_needed  (bool)   Enable compaction
     *   compact_keep_recent (int)   Recent messages to keep (default 6)
     *   compact_model       (string) Cheap model for summarization
     *
     * Returns updated $params with compacted messages.
     */
    private function compactMessagesIfNeeded(array $params, string $model, ?int $contextWindowOverride = null): array
    {
        if (empty($params['compact_if_needed'])) {
            return $params;
        }

        $messages = $params['messages'] ?? [];
        if (empty($messages)) {
            return $params;
        }

        // Estimate current token count
        $messagesText = $this->messagesToText($messages);
        $currentTokens = $this->estimateTokens($messagesText, $model);

        // Get context window for this model (allow override for testing)
        $contextWindow = $contextWindowOverride ?? ($this->getModelContextSpec($model)['context'] ?? 128000);
        $threshold = (int) ($contextWindow * 0.8);

        if ($currentTokens < $threshold) {
            $this->emDebug("Compaction: skipping ({$currentTokens} tokens < {$threshold} threshold)");
            return $params;
        }

        $keepRecent = (int) ($params['compact_keep_recent'] ?? 6);
        $summarizeModel = $params['compact_model'] ?? $model;

        // Don't compact if too few messages
        if (count($messages) <= $keepRecent + 2) {
            return $params;
        }

        $this->emDebug("Compaction: triggering ({$currentTokens} tokens > {$threshold} threshold)", [
            'total_messages' => count($messages),
            'keep_recent' => $keepRecent,
            'summarize_model' => $summarizeModel,
        ]);

        // Split into old + recent
        $recentMessages = array_slice($messages, -$keepRecent);
        $oldMessages = array_slice($messages, 0, count($messages) - $keepRecent);

        // Separate system messages from conversation messages
        $systemMessages = [];
        $conversationMessages = [];
        foreach ($oldMessages as $msg) {
            if (($msg['role'] ?? '') === 'system') {
                $systemMessages[] = $msg;
            } else {
                $conversationMessages[] = $msg;
            }
        }

        if (empty($conversationMessages)) {
            return $params;
        }

        // Summarize old conversation messages
        $oldText = $this->messagesToText($conversationMessages);
        $summaryPrompt = "Concisely summarize this conversation in 3-5 sentences. "
            . "Preserve key facts, decisions, and context. Be brief but comprehensive.\n\n"
            . $oldText;

        try {
            // Use correct token param for this model
            $o1Models = ['o1', 'o3-mini', 'o3', 'o4-mini', 'gpt-5-6-sol', 'gpt-5-6-luna', 'gpt-5-6-terra'];
            $tokenParam = in_array($summarizeModel, $o1Models)
                ? 'max_completion_tokens'
                : 'max_tokens';

            // Same policy as the conversation being summarized. If the summarize
            // model isn't configured under it, callLLMOnce throws and compaction is
            // skipped below — the history is never sent to the other model set.
            $summaryResponse = $this->callLLMOnce($summarizeModel, [
                'messages' => [
                    ['role' => 'user', 'content' => $summaryPrompt]
                ],
                $tokenParam => 500,
                'token_policy' => $params['token_policy'] ?? $this->activeTokenPolicy ?? TokenPolicy::STANDARD,
            ], null);

            $summary = trim($summaryResponse['content'] ?? '');

            if (empty($summary)) {
                $this->emDebug("Compaction: summarization returned empty, skipping");
                return $params;
            }

            // Rebuild messages: system + summary + recent
            $summaryMsg = [
                'role' => 'system',
                'content' => "Previous conversation summary: {$summary}"
            ];

            $compactedMessages = array_merge(
                $systemMessages,
                [$summaryMsg],
                $recentMessages
            );

            // Log metrics
            $compactedText = $this->messagesToText($compactedMessages);
            $compactedTokens = $this->estimateTokens($compactedText, $model);
            $this->emDebug("Compaction: complete", [
                'before_tokens' => $currentTokens,
                'after_tokens' => $compactedTokens,
                'reduction' => round((1 - $compactedTokens / $currentTokens) * 100, 1) . '%',
                'before_messages' => count($messages),
                'after_messages' => count($compactedMessages),
            ]);

            $params['messages'] = $compactedMessages;
            return $params;

        } catch (\Throwable $e) {
            $this->emDebug("Compaction: summarization failed, proceeding without: " . $e->getMessage());
            return $params;
        }
    }

    /**
     * Get context window spec for a model.
     */
    private function getModelContextSpec(string $model): array
    {
        $specs = [
            'o1'               => ['context' => 200000],
            'o3'               => ['context' => 200000],
            'o3-mini'          => ['context' => 200000],
            'o4-mini'          => ['context' => 200000],
            'gpt-4o'           => ['context' => 128000],
            'gpt-4-1'          => ['context' => 1000000],
            'gpt-4-1-nano'     => ['context' => 1000000],
            'gpt-5'            => ['context' => 400000],
            'gpt-5-4'          => ['context' => 400000],
            'gpt-5-4-nano'     => ['context' => 400000],
            'gpt-5-6-sol'      => ['context' => 400000],
            'gpt-5-6-luna'     => ['context' => 400000],
            'gpt-5-6-terra'    => ['context' => 400000],
            'gpt-5-nano'       => ['context' => 400000],
            'grok-3'           => ['context' => 128000],
            'grok-3-mini'      => ['context' => 128000],
            'deepseek'         => ['context' => 128000],
            'deepseek-r1'      => ['context' => 128000],
            'claude'           => ['context' => 200000],
            'claude-sonnet-3.5' => ['context' => 200000],
            'claude-sonnet-3.7' => ['context' => 200000],
            'claude-haiku'     => ['context' => 200000],
            'claude-haiku-4.5' => ['context' => 200000],
            'claude-opus-4'    => ['context' => 200000],
            'claude-opus-4-1'  => ['context' => 200000],
            'claude-opus-4-6'  => ['context' => 200000],
            'claude-opus-4-7'  => ['context' => 200000],
            'claude-sonnet-4'  => ['context' => 200000],
            'claude-sonnet-4-5' => ['context' => 200000],
            'claude-sonnet-4-6' => ['context' => 200000],
            'gemini-2.5-pro'   => ['context' => 1000000],
            'gemini-2.5-flash' => ['context' => 1000000],
            'llama3370b'       => ['context' => 128000],
            'llama-Maverick'   => ['context' => 128000],
            'llama-4-scout'    => ['context' => 128000],
        ];
        return $specs[$model] ?? ['context' => 128000];
    }

    private function estimateTokens(string $text, string $model): int {
        try {
            $this->encoderProvider ??= new EncoderProvider();
            $encoder = $this->encoderProvider->getForModel($model);
            return count($encoder->encode($text));
        } catch (\Throwable $e) {
            // Fallback for models not in tiktoken registry (e.g. llama-Maverick)
            return (int)(mb_strlen($text) / 4);
        }
    }

    private function computeDynamicMaxTokens(string $model, string $prompt): array {
        $modelSpecs = [
            'o1' => [
                'context' => 200000,
                'output_max' => 100000,
                'param' => 'max_completion_tokens',
                'buffer' => 25000
            ],
            'o3' => [
                'context' => 200000,
                'output_max' => 100000,
                'param' => 'max_completion_tokens',
                'buffer' => 25000
            ],
            'o3-mini' => [
                'context' => 200000,
                'output_max' => 100000,
                'param' => 'max_completion_tokens',
                'buffer' => 25000
            ],
            'o4-mini' => [
                'context' => 200000,
                'output_max' => 100000,
                'param' => 'max_completion_tokens',
                'buffer' => 25000
            ],
            'gpt-4-1' => [
                'context' => 1000000,
                'output_max' => 128000,
                'param' => 'max_tokens',
                'buffer' => 2000
            ],
            'gpt-4-1-nano' => [
                'context' => 1000000,
                'output_max' => 128000,
                'param' => 'max_tokens',
                'buffer' => 2000
            ],
            'gpt-5' => [
                'context' => 400000,
                'output_max' => 100000,
                'param' => 'max_completion_tokens',
                'buffer' => 25000
            ],
            'gpt-5-4' => [
                'context' => 400000,
                'output_max' => 128000,
                'param' => 'max_tokens',
                'buffer' => 2000
            ],
            'gpt-5-4-nano' => [
                'context' => 400000,
                'output_max' => 128000,
                'param' => 'max_tokens',
                'buffer' => 2000
            ],
            'gpt-5-6-sol' => [
                'context' => 400000,
                'output_max' => 100000,
                'param' => 'max_completion_tokens',
                'buffer' => 25000
            ],
            'gpt-5-6-luna' => [
                'context' => 400000,
                'output_max' => 100000,
                'param' => 'max_completion_tokens',
                'buffer' => 25000
            ],
            'gpt-5-6-terra' => [
                'context' => 400000,
                'output_max' => 100000,
                'param' => 'max_completion_tokens',
                'buffer' => 25000
            ],
            'claude' => [
                'context' => 200000,
                'output_max' => 8192,
                'param' => 'max_tokens',
                'buffer' => 2000
            ],
            'claude-sonnet-3.5' => [
                'context' => 200000,
                'output_max' => 8192,
                'param' => 'max_tokens',
                'buffer' => 2000
            ],
            'claude-sonnet-3.7' => [
                'context' => 200000,
                'output_max' => 8192,
                'param' => 'max_tokens',
                'buffer' => 2000
            ],
            'claude-haiku-4.5' => [
                'context' => 200000,
                'output_max' => 8192,
                'param' => 'max_tokens',
                'buffer' => 2000
            ],
            'claude-opus-4' => [
                'context' => 200000,
                'output_max' => 32000,
                'param' => 'max_tokens',
                'buffer' => 2000
            ],
            'claude-sonnet-4' => [
                'context' => 200000,
                'output_max' => 16000,
                'param' => 'max_tokens',
                'buffer' => 2000
            ]
        ];
        $spec = $modelSpecs[$model] ?? null;
        if (!$spec) return ['max_tokens', 16384, 0];

        $promptTokens = $this->estimateTokens($prompt, $model);
        $available = $spec['context'] - $promptTokens - $spec['buffer'];
        $final = min($available, $spec['output_max']);
        $final = max(1024, $final);  // Min fallback

        // Metadata only (computed token ceiling) — no prompt/response content. Routed
        // through emDebug so it respects debug-mode gating instead of always-on error_log.
        $this->emDebug("computeDynamicMaxTokens", [
            'model'         => $model,
            'prompt_tokens' => $promptTokens,
            'final_max'     => $final,
        ]);

        return [$spec['param'], $final, $promptTokens];
    }


// OPTIONAL FOR LATER , MODEL SPECIFIC STUFF
private function modelSupportsNativeTools(string $model): bool
{
    // Intentionally disabled for initial agent rollout
    return false;

    /*
    $raw = (string) ($this->getSystemSetting('agent_tools_native_models') ?? '');
    $allow = array_filter(array_map('trim', explode(',', $raw)));
    if (empty($allow)) return false;

    return in_array($model, $allow, true);
    */
}

private function toOpenAIToolsShape(array $tools): array
{
    // Intentionally disabled for initial agent rollout
    return [];

    /*
    $out = [];
    foreach ($tools as $t) {
        if (empty($t['name'])) continue;
        $out[] = [
            'type' => 'function',
            'function' => [
                'name' => $t['name'],
                'description' => $t['description'] ?? '',
                'parameters' => $t['parameters'] ?? [
                    'type' => 'object',
                    'properties' => []
                ]
            ]
        ];
    }
    return $out;
    */
}
// OPTIONAL FOR LATER , MODEL SPECIFIC STUFF


    /**
     * Build a PHI-safe metadata summary of an LLM request/response for debug logging.
     *
     * Prompt and response BODIES may contain PHI, so they must never be written to the
     * debug logs. This helper extracts only non-content metadata (token usage, model,
     * role, finish reason, structural shape and content length) that is safe to log.
     *
     * @param mixed $response Raw or normalized response from an LLM adapter.
     * @return array Safe-to-log metadata describing the response (no content body).
     */
    private function summarizeResponseForDebug($response): array
    {
        // Non-array payloads carry no structured content to redact; report the type only.
        if (!is_array($response)) {
            return ['type' => gettype($response)];
        }

        // Resolve the message content from either the normalized shape ('content')
        // or the raw OpenAI-compatible shape ('choices[0].message.content') WITHOUT
        // ever including the text itself in the returned metadata.
        $content = $response['content']
            ?? ($response['choices'][0]['message']['content'] ?? null);

        return [
            // Key names only (e.g. "content", "usage") — never their values.
            'top_level_keys' => array_keys($response),
            'model'          => $response['model'] ?? null,
            'role'           => $response['role']
                ?? ($response['choices'][0]['message']['role'] ?? null),
            'finish_reason'  => $response['choices'][0]['finish_reason'] ?? null,
            // Token counts are metadata, not content — safe to log.
            'usage'          => $response['usage'] ?? null,
            'choices_count'  => isset($response['choices']) && is_array($response['choices'])
                ? count($response['choices'])
                : null,
            // Length and shape signals only — never the text itself.
            'content_length' => is_string($content) ? strlen($content) : null,
            'has_tool_call'  => is_string($content) && str_contains($content, 'tool_call'),
        ];
    }

    private function normalizeResponse($response, $model)
    {
        $normalized = [];

        $this->emDebug("normalizeResponse called", [
            'model' => $model,
            'has_data' => isset($response['data']),
            'has_choices' => isset($response['choices'])
        ]);

        // Embedding models return their own format - pass through unchanged
        if (str_contains($model, 'embed') || $model === 'ada-002') {
            $this->emDebug("Passing through embedding response unchanged for model: {$model}");
            return $response;
        }

        if (str_starts_with($model, 'claude') || str_contains($model, 'anthropic')) {
            $normalized['content'] = $response['content'][0]['text'] ?? '';
            $normalized['role'] = $response['role'] ?? 'assistant';
            $normalized['model'] = $response['model'] ?? 'claude';
            $normalized['usage'] = [
                'prompt_tokens' => $response['usage']['input_tokens'] ?? 0,
                'completion_tokens' => $response['usage']['output_tokens'] ?? 0,
                'total_tokens' => ($response['usage']['input_tokens'] ?? 0) + ($response['usage']['output_tokens'] ?? 0)
            ];
        } elseif (str_contains($model, 'tts')) {
            $normalized['audio_base64'] = $response['audio_base64'] ?? '';
            $normalized['content_type'] = $response['content_type'] ?? 'audio/mpeg';
            $normalized['model'] = $model;
        } elseif (isset($response['choices'][0]['message']['content'])) {
            // OpenAI-compatible format (catches all chat models dynamically)
            $normalized['content'] = $response['choices'][0]['message']['content'] ?? '';
            $decoded = json_decode($normalized['content'], true);
            if (json_last_error() === JSON_ERROR_NONE && is_array($decoded)) {
                $normalized['structured_output'] = $decoded;
                // Only mark preserve_structure when the JSON looks like a genuine
                // tool-call / structured-output payload, NOT when it's an agent-mode
                // response envelope ({final_answer|tool_call|thinking}). The agent
                // schema is consumed by runAgentLoop before sanitizeOutputForUI,
                // so by the time we get here it must already be unwrapped — any
                // remaining JSON is either an unwrapped envelope (bug) or
                // legitimate structured output. Inspect the shape.
                $isAgentEnvelope = isset($decoded['final_answer'])
                    || isset($decoded['tool_call'])
                    || isset($decoded['thinking']);
                if (!$isAgentEnvelope) {
                    $normalized['preserve_structure'] = true;
                }
            }
            $normalized['role'] = $response['choices'][0]['message']['role'] ?? 'assistant';
            $normalized['model'] = $response['model'] ?? $model;
            $normalized['usage'] = [
                'prompt_tokens' => $response['usage']['prompt_tokens'] ?? 0,
                'completion_tokens' => $response['usage']['completion_tokens'] ?? 0,
                'total_tokens' => $response['usage']['total_tokens'] ?? 0
            ];
        } else {
            $normalized = $response;
        }

        // $this->emDebug("normalized responseData", $normalized);
        return $normalized;
    }

    /**
     * Build a rehydrated session from dedicated one-row-per-turn records.
     * Each row already holds the real user prompt and final assistant answer,
     * so this is a straight map — no filtering of agent-loop bookkeeping, and
     * errored turns keep their user prompt (they are NOT skipped).
     */
    private function buildSessionFromTurns($session_id, $pid, array $turnLogs)
    {
        $messages = [];
        $models = [];
        $total_tokens = 0;
        $first_timestamp = null;
        $last_timestamp = null;

        foreach ($turnLogs as $index => $log) {
            if (!empty($log['user_message'])) {
                $messages[] = [
                    'role' => 'user',
                    'content' => $log['user_message'],
                    'turn' => $index + 1,
                ];
            }
            if (!empty($log['assistant_response'])) {
                $messages[] = [
                    'role' => 'assistant',
                    'content' => $log['assistant_response'],
                    'turn' => $index + 1,
                    'tools_used' => $log['tools_used'] ?? null,
                ];
            }
            if (!empty($log['model'])) $models[$log['model']] = true;
            if (!empty($log['usage']['total_tokens'])) $total_tokens += $log['usage']['total_tokens'];
            if ($first_timestamp === null) $first_timestamp = $log['timestamp'];
            $last_timestamp = $log['timestamp'];
        }

        return [
            'session_id' => $session_id,
            'messages' => $messages,
            'metadata' => [
                'project_id' => $pid,
                'start_time' => $first_timestamp,
                'end_time' => $last_timestamp,
                'duration_seconds' => $first_timestamp && $last_timestamp ?
                    strtotime($last_timestamp) - strtotime($first_timestamp) : 0
            ],
            'stats' => [
                'total_turns' => count($turnLogs),
                'total_tokens' => $total_tokens,
                'models_used' => array_keys($models)
            ]
        ];
    }

    /**
     * Log ONE conversation turn (real user prompt + final assistant answer) as
     * a dedicated record, distinct from the per-LLM-call audit rows written by
     * logInteraction(). rehydrateProjectSession reads THESE rows first to
     * rebuild chat history reliably.
     *
     * Two-store design (intentional, not redundant to remove): logInteraction()
     * still writes its per-LLM-call rows — those remain the AUDIT trail (every
     * model call, consumed by getSecureChatLogsBySession and admin tooling).
     * CappyChatTurn rows are the REHYDRATION source (one clean row per turn).
     * They overlap in content but serve different consumers; do not drop the
     * per-call rows on the assumption turn rows supersede them.
     *
     * Skipped when there is no session_id, or when the caller opts out via
     * $requestData['log_turn'] === false (e.g. the client-side compaction
     * summary call, which reuses the session id but is not a real user turn).
     */
    private function logConversationTurn($project_id, $requestData, $responseData, $username = null)
    {
        if (empty($requestData['session_id'])) {
            $this->emDebug("logConversationTurn SKIP", ['pid' => $project_id, 'reason' => 'no_session_id']);
            return;
        }
        if (array_key_exists('log_turn', $requestData) && $requestData['log_turn'] === false) {
            $this->emDebug("logConversationTurn SKIP", ['pid' => $project_id, 'session_id' => $requestData['session_id'], 'reason' => 'log_turn_false']);
            return;
        }

        // Real user prompt = last user message in the INCOMING messages. The
        // agent loop's injected TOOL RESULT messages live on a local copy inside
        // runAgentLoop, so $requestData['messages'] here is still the clean UI
        // context.
        $userMessage = '';
        if (!empty($requestData['messages']) && is_array($requestData['messages'])) {
            foreach (array_reverse($requestData['messages']) as $msg) {
                if (($msg['role'] ?? '') === 'user') {
                    $userMessage = (string)($msg['content'] ?? '');
                    break;
                }
            }
        }

        $isError = !empty($responseData['error']);
        $assistant = $responseData['content']
            ?? ($responseData['choices'][0]['message']['content'] ?? '');

        // Skip empty turns — a row with neither a user prompt nor an assistant
        // answer (e.g. an embedding/structured response that happens to carry a
        // session_id) is junk that rehydrates to a blank bubble.
        if (trim((string)$userMessage) === '' && trim((string)$assistant) === '') {
            $this->emDebug("logConversationTurn SKIP", ['pid' => $project_id, 'session_id' => $requestData['session_id'], 'reason' => 'empty_turn']);
            return;
        }

        // TEMP DIAGNOSTIC (PHI-safe: counts/ids/flags only, never content).
        // Confirms which session_id a turn is written under, to compare against
        // what rehydrate later queries. Remove once the write/read id mismatch
        // is pinpointed.
        $this->emDebug("logConversationTurn WRITE", [
            'pid'           => $project_id,
            'session_id'    => $requestData['session_id'],
            'username'      => $username,
            'has_user'      => trim((string)$userMessage) !== '',
            'has_assistant' => trim((string)$assistant) !== '',
            'is_error'      => $isError,
        ]);

        $atomicPayload = [
            'project_id'         => $project_id,
            'session_id'         => $requestData['session_id'],
            // Owner of the turn. rehydrateProjectSession filters by this so one
            // user can't rebuild another user's chat from a guessed session_id.
            'username'           => $username !== null ? (string)$username : null,
            'model'              => $responseData['model'] ?? ($requestData['model'] ?? 'unknown'),
            // Which AI Hub model set served the turn. Never the token itself.
            'token_policy'       => $requestData['token_policy'] ?? null,
            'timestamp'          => date('Y-m-d H:i:s'),
            'user_message'       => $userMessage,
            'assistant_response' => $assistant,
        ];
        if (!empty($responseData['tools_used'])) {
            $atomicPayload['tools_used'] = $responseData['tools_used'];
        }
        if (!empty($responseData['usage'])) {
            $atomicPayload['usage'] = $responseData['usage'];
        }
        if ($isError) {
            $atomicPayload['error'] = true;
        }

        $action = new SecureChatLog($this);
        $action->setValue('message', json_encode($atomicPayload));
        $action->setValue('record', $isError ? 'CappyChatTurnError' : 'CappyChatTurn');
        if ($project_id) {
            $action->setValue('project_id', $project_id);
        }
        $action->setValue('session_id', $requestData['session_id']);
        if (!empty($atomicPayload['model'])) {
            $action->setValue('model', $atomicPayload['model']);
        }
        $action->save();
    }

    private function logInteraction($project_id, $requestData, $responseData, $username = null)
    {
        // Extract atomic log data - only the current turn, not full conversation history
        $atomicPayload = [
            'project_id' => $project_id,
            'session_id' => $requestData['session_id'] ?? null,
            // REDCap login of the requesting user (not PHI). Older SecureChatLog
            // rows predate this, so the usage dashboard shows them as "not logged".
            'username' => $username !== null ? (string)$username : null,
            'model' => $responseData['model'] ?? ($requestData['model'] ?? 'unknown'),
            'token_policy' => $requestData['token_policy'] ?? null,
            'timestamp' => date('Y-m-d H:i:s')
        ];

        // Extract the last user message from the messages array
        if (!empty($requestData['messages']) && is_array($requestData['messages'])) {
            // Find the last user message (skip system messages)
            $userMessages = array_filter($requestData['messages'], function($msg) {
                return ($msg['role'] ?? '') === 'user';
            });
            $lastUserMessage = !empty($userMessages) ? array_values($userMessages)[count($userMessages) - 1] : null;

            if ($lastUserMessage) {
                $atomicPayload['user_message'] = $lastUserMessage['content'] ?? '';
            }
        }

        // Extract assistant response
        if (!empty($responseData)) {
            // Handle normalized response structure
            if (isset($responseData['content'])) {
                $atomicPayload['assistant_response'] = $responseData['content'];
            } elseif (isset($responseData['choices'][0]['message']['content'])) {
                $atomicPayload['assistant_response'] = $responseData['choices'][0]['message']['content'];
            }

            // Extract usage data
            if (isset($responseData['usage'])) {
                $atomicPayload['usage'] = $responseData['usage'];
            }

            // Extract any tool usage metadata
            if (isset($responseData['tools_used'])) {
                $atomicPayload['tools_used'] = $responseData['tools_used'];
            }
        }

        $action = new SecureChatLog($this);
        $action->setValue('message', json_encode($atomicPayload));
        $action->setValue('record', 'SecureChatLog');
        if ($project_id) {
            $action->setValue('project_id', $project_id);
        }
        if (!empty($requestData['session_id'])) {
            $action->setValue('session_id', $requestData['session_id']);
        }
        $model = $responseData['model'] ?? ($requestData['model'] ?? null);
        if (!empty($model)) {
            $action->setValue('model', $model);
        }
        $action->save();
    }

    private function logErrorInteraction($project_id, $requestData, $error)
    {
        // Extract atomic error log data
        $atomicPayload = [
            'project_id' => $project_id,
            'session_id' => $requestData['session_id'] ?? null,
            'model' => $requestData['model'] ?? 'unknown',
            'token_policy' => $requestData['token_policy'] ?? null,
            'timestamp' => date('Y-m-d H:i:s'),
            'error' => true,
            'error_type' => $error['type'] ?? 'UNKNOWN',
            'error_message' => $error['message'] ?? 'Unknown error'
        ];

        // Extract the last user message that caused the error
        if (!empty($requestData['messages']) && is_array($requestData['messages'])) {
            $userMessages = array_filter($requestData['messages'], function($msg) {
                return ($msg['role'] ?? '') === 'user';
            });
            $lastUserMessage = !empty($userMessages) ? array_values($userMessages)[count($userMessages) - 1] : null;

            if ($lastUserMessage) {
                $atomicPayload['user_message'] = $lastUserMessage['content'] ?? '';
            }
        }

        $action = new SecureChatLog($this);
        $action->setValue('message', json_encode($atomicPayload));
        $action->setValue('record', 'SecureChatLogError');
        if ($project_id) {
            $action->setValue('project_id', $project_id);
        }
        if (!empty($requestData['session_id'])) {
            $action->setValue('session_id', $requestData['session_id']);
        }
        $model = $requestData['model'] ?? null;
        if (!empty($model)) {
            $action->setValue('model', $model);
        }
        $action->save();
    }

    public function extractResponseText($response)
    {
        // Return structured output as-is if available (already JSON)
        if (isset($response['structured_output'])) {
            return $response['structured_output'];
        }
        return $response['content'] ?? json_encode($response);
    }


    public function extractUsageTokens($response)
    {
        return $response['usage'] ?? ['prompt_tokens' => 0, 'completion_tokens' => 0, 'total_tokens' => 0];
    }

    public function extractMetaData($response)
    {
        return [
            'id' => $response['id'] ?? 'N/A',
            'object' => $response['object'] ?? 'N/A',
            'created' => $response['created'] ?? 'N/A',
            'model' => $response['model'] ?? 'N/A',
            'usage' => $response['usage'] ?? 'N/A'
        ];
    }

    public function getGuzzleClient()
    {
        if (!$this->guzzleClient) {
            $options = [
                'timeout' => 30,
                'connect_timeout' => 10,
                'verify' => false
            ];

            // Apply DNS override if configured (for site-to-site VPN routing)
            $dnsOverrideIp = $this->getSystemSetting('apim_dns_override_ip');
            if (!empty($dnsOverrideIp)) {
                $options['curl'] = [
                    CURLOPT_RESOLVE => ["aihubapi.stanfordhealthcare.org:443:{$dnsOverrideIp}"],
                ];
            }

            $this->guzzleClient = new Client($options);
        }
        return $this->guzzleClient;
    }

    public function setGuzzleTimeout(float $timeout)
    {
        $this->guzzleTimeout = $timeout;
    }

    public function getGuzzleTimeout(): float
    {
        return $this->guzzleTimeout;
    }

    /**
     * This is the primary ajax handler for JSMO calls
     * @param $action
     * @param $payload
     * @param $project_id
     * @return array|array[]|bool
     * @throws Exception
     */

    /**
     * Config-dialog save validation (system and project): token policy values,
     * and that every model row ends up with a token.
     */
    public function validateSettings($settings)
    {
        return TokenPolicy::validateSettings(is_array($settings) ? $settings : []);
    }

    /**
     * Error response for the REST actions when the token policy refuses a call.
     */
    private function apiTokenPolicyError(string $message, bool $claudeFormat): array
    {
        // "error" must be the first key: RestUtility::sendResponse() re-wraps any
        // non-200 JSON body that doesn't start with {"error" into {"error": "<body>"}.
        $body = $claudeFormat
            ? ["error" => ["type" => "token_policy_error", "message" => $message], "type" => "error"]
            : ["error" => $message, "type" => "TOKEN_POLICY_ERROR"];
        return [
            "status"  => 400,
            "body"    => json_encode($body),
            "headers" => ["Content-Type" => "application/json"]
        ];
    }

    public function redcap_module_api($action = null, $payload = [], $api_project_id = null)
    {
        if (empty($action) && isset($_POST['action'])) {
            $action = $_POST['action'];
        }

        // Normalize payload from JSON or POST
        if (empty($payload)) {
            $raw = file_get_contents("php://input");
            $decoded = $raw ? json_decode($raw, true) : [];
            $payload = (json_last_error() === JSON_ERROR_NONE) ? $decoded : $_POST;
        }

        switch ($action) {
            case "callAI":
                $prompt = $payload['prompt'] ?? null;
                $model = $payload['model'] ?? 'deepseek';
                $temperature = isset($payload['temperature']) ? (float)$payload['temperature'] : 0.2;
                $max_tokens = isset($payload['max_tokens']) ? (int)$payload['max_tokens'] : 8000;
                $json_schema = $payload['json_schema'] ?? null;

                if (!$prompt) {
                    return [
                        "status"  => 400,
                        "body"    => json_encode(["error" => "Missing prompt"]),
                        "headers" => ["Content-Type" => "application/json"]
                    ];
                }

                $params = [
                    'messages' => [['role' => 'user', 'content' => $prompt]],
                    'temperature' => $temperature,
                    'max_tokens' => $max_tokens
                ];

                // Resolved against the API token's project so its minimum policy
                // holds for REST callers too (callAI() below runs without a pid).
                try {
                    $params['token_policy'] = $this->resolveTokenPolicy($payload['token_policy'] ?? null, $api_project_id);
                } catch (TokenPolicyException $e) {
                    return $this->apiTokenPolicyError($e->getMessage(), false);
                }

                if ($json_schema !== null) {
                    $decoded = json_decode($json_schema, true);
                    if (json_last_error() === JSON_ERROR_NONE) {
                        $params['json_schema'] = $decoded;
                    }
                }

                $result = $this->callAI($model, $params);

                if (($result['type'] ?? null) === 'TOKEN_POLICY_ERROR') {
                    return $this->apiTokenPolicyError($result['message'] ?? 'Refused by token policy', false);
                }

                $response = [
                    'status' => 'success',
                    'model' => $model,
                    'token_policy' => $params['token_policy'],
                    'content' => $this->extractResponseText($result),
                    'usage' => $this->extractUsageTokens($result)
                ];

                return [
                    "status"  => 200,
                    "body"    => json_encode($response),
                    "headers" => ["Content-Type" => "application/json"]
                ];

            case "messages":
                // Claude Messages API compatibility endpoint
                // Accepts: {model, messages, max_tokens, temperature, system, top_p, stop}
                // Returns: Claude Messages API format response

                $model = $payload['model'] ?? 'o3-mini';
                $messages = $payload['messages'] ?? null;

                // Handle messages as JSON string (from form-encoded POST)
                if (is_string($messages)) {
                    $decoded = json_decode($messages, true);
                    if (json_last_error() === JSON_ERROR_NONE && is_array($decoded)) {
                        $messages = $decoded;
                    }
                }

                $max_tokens = isset($payload['max_tokens']) ? (int)$payload['max_tokens'] : 4096;
                $temperature = isset($payload['temperature']) ? (float)$payload['temperature'] : null;
                $top_p = isset($payload['top_p']) ? (float)$payload['top_p'] : null;
                $stop = $payload['stop'] ?? null;
                $system = $payload['system'] ?? null;

                if (!$messages || !is_array($messages)) {
                    return [
                        "status"  => 400,
                        "body"    => json_encode([
                            "type" => "error",
                            "error" => [
                                "type" => "invalid_request_error",
                                "message" => "Missing or invalid 'messages' field"
                            ]
                        ]),
                        "headers" => ["Content-Type" => "application/json"]
                    ];
                }

                // Map Claude model names to SecureChatAI aliases
                $modelMap = [
                    'claude-3-7-sonnet-20250219' => 'claude',
                    'claude-3-5-sonnet-20241022' => 'claude',
                    'claude-sonnet-4' => 'claude-sonnet-4',
                    'gpt-4-1' => 'gpt-4-1',
                    'o1' => 'o1',
                    'o3-mini' => 'o3-mini',
                    'gpt-5' => 'gpt-5',
                    'llama3370b' => 'llama3370b',
                    'deepseek' => 'deepseek'
                ];
                $mappedModel = $modelMap[$model] ?? $model;

                // Prepend system message if provided
                if ($system) {
                    array_unshift($messages, [
                        'role' => 'system',
                        'content' => $system
                    ]);
                }

                // Build params for callAI()
                $params = ['messages' => $messages];

                try {
                    $params['token_policy'] = $this->resolveTokenPolicy($payload['token_policy'] ?? null, $api_project_id);
                } catch (TokenPolicyException $e) {
                    return $this->apiTokenPolicyError($e->getMessage(), true);
                }

                if ($temperature !== null) $params['temperature'] = $temperature;
                if ($top_p !== null) $params['top_p'] = $top_p;
                if ($stop !== null) $params['stop'] = $stop;

                // Handle max_tokens vs max_completion_tokens
                if (in_array($mappedModel, ['o1', 'o3-mini', 'gpt-5', 'gpt-5-6-sol', 'gpt-5-6-luna', 'gpt-5-6-terra'])) {
                    $params['max_completion_tokens'] = $max_tokens;
                } else {
                    $params['max_tokens'] = $max_tokens;
                }

                // Call the AI (no project_id for external API calls)
                $result = $this->callAI($mappedModel, $params, null);

                if (($result['type'] ?? null) === 'TOKEN_POLICY_ERROR') {
                    return $this->apiTokenPolicyError($result['message'] ?? 'Refused by token policy', true);
                }

                // Check for errors
                if (isset($result['error']) && $result['error']) {
                    return [
                        "status"  => 500,
                        "body"    => json_encode([
                            "type" => "error",
                            "error" => [
                                "type" => $result['type'] ?? "api_error",
                                "message" => $result['message'] ?? "Internal error"
                            ]
                        ]),
                        "headers" => ["Content-Type" => "application/json"]
                    ];
                }

                // Transform to Claude Messages API response format
                $content = $this->extractResponseText($result);
                $usage = $this->extractUsageTokens($result);

                // Determine stop_reason
                $stop_reason = "end_turn";
                if (isset($result['usage']['completion_tokens']) &&
                    isset($params['max_tokens']) &&
                    $result['usage']['completion_tokens'] >= $params['max_tokens']) {
                    $stop_reason = "max_tokens";
                }

                $claudeResponse = [
                    "id" => "msg_" . uniqid(),
                    "type" => "message",
                    "role" => "assistant",
                    "content" => [
                        [
                            "type" => "text",
                            "text" => is_string($content) ? $content : json_encode($content)
                        ]
                    ],
                    "model" => $model,  // Echo back the requested model name
                    "stop_reason" => $stop_reason,
                    "stop_sequence" => null,
                    "usage" => [
                        "input_tokens" => $usage['prompt_tokens'] ?? 0,
                        "output_tokens" => $usage['completion_tokens'] ?? 0
                    ]
                ];

                return [
                    "status"  => 200,
                    "body"    => json_encode($claudeResponse),
                    "headers" => ["Content-Type" => "application/json"]
                ];

            case "getSession":
                $session_id = $payload['session_id'] ?? null;
                $project_id = $payload['project_id'] ?? null;

                if (!$session_id) {
                    return [
                        "status"  => 400,
                        "body"    => json_encode(["error" => "Missing session_id"]),
                        "headers" => ["Content-Type" => "application/json"]
                    ];
                }

                // Use the same rehydration path as the chatbot (prefers clean
                // CappyChatTurn rows, backfills pre-turn legacy history) so the
                // two "rehydrate a session" entry points don't diverge. This is
                // the token-API path (no browser user), so it is not user-scoped
                // — the API token is the authorization boundary here.
                $session = $this->rehydrateProjectSession($session_id, $project_id);

                return [
                    "status"  => 200,
                    "body"    => json_encode($session),
                    "headers" => ["Content-Type" => "application/json"]
                ];

            default:
                return [
                    "status"  => 400,
                    "body"    => json_encode(["error" => "Action $action not defined"]),
                    "headers" => ["Content-Type" => "application/json"]
                ];
        }
    }
}
?>