Skip to content

feat(tts): play TTS for visible messages with a keyboard shortcut #426

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chriswritescode-dev merged 4 commits into from
Oct 14, 2026
Merged

feat(tts): play TTS for visible messages with a keyboard shortcut #426

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7cb6c07
feat(tts): play TTS for visible messages with a keyboard shortcut
chriswritescode-dev Oct 13, 2026
eafa894
fix(tts): exclude covered messages and skip auto-play during shortcut…
chriswritescode-dev Oct 14, 2026
8a893d0
refactor(tts): read visible replies when floating button is tapped wh…
chriswritescode-dev Oct 14, 2026
cc0e136
test(e2e): wait for TTS synthesis to settle before asserting exclusions
chriswritescode-dev Oct 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
164 changes: 14 additions & 150 deletions e2e/inline-images.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
import { execFileSync } from 'node:child_process'
import fs from 'node:fs'
import http from 'node:http'
import path from 'node:path'
import { request as playwrightRequest, type APIRequestContext, type Locator } from '@playwright/test'
import { type Locator } from '@playwright/test'
import { test, expect } from './fixtures'
import { BASE_URL, E2E_TMP, STORAGE_STATE } from './env'
import { createMockRepo, createSession, startMockProvider, type MockProvider, type MockRepo } from './mockProvider'

const PNG_BASE64 =
'iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAFElEQVQI12P8z8Dwn4GBgYGJAQoAHxcCAmep4oIAAAAASUVORK5CYII='
Expand All @@ -14,17 +12,8 @@ const PNG_BUFFER = Buffer.from(PNG_BASE64, 'base64')
const WEBP_BUFFER = Buffer.from(WEBP_BASE64, 'base64')

const MOCK_PROVIDER_PORT = 5565
const MOCK_MODEL_ID = 'mock-model'

let mockReply = ''
let mockServer: http.Server | undefined

interface MockRepo {
id: number
fullPath: string
sessionId: string
}

let mockProvider: MockProvider
let mockRepo: MockRepo

function rawFileUrl(filePath: string): string {
Expand All @@ -39,144 +28,19 @@ async function expectImageLoaded(image: Locator): Promise<void> {
.toBeGreaterThan(0)
}

function handleMockRequest(request: http.IncomingMessage, response: http.ServerResponse): void {
const chunks: Buffer[] = []
request.on('data', (chunk: Buffer) => chunks.push(chunk))
request.on('end', () => {
const url = request.url ?? ''
if (request.method === 'GET' && url.startsWith('/v1/models')) {
response.writeHead(200, { 'Content-Type': 'application/json' })
response.end(
JSON.stringify({
object: 'list',
data: [{ id: MOCK_MODEL_ID, object: 'model', created: 0, owned_by: 'mock' }],
}),
)
return
}

if (request.method === 'POST' && url.startsWith('/v1/chat/completions')) {
let body: { stream?: boolean } = {}
try {
body = JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}')
} catch {
body = {}
}

const id = 'chatcmpl-mock'
const created = Math.floor(Date.now() / 1000)
const usage = { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }

if (body.stream) {
response.writeHead(200, {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
Connection: 'keep-alive',
})
const base = { id, object: 'chat.completion.chunk', created, model: MOCK_MODEL_ID }
response.write(
`data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: { role: 'assistant', content: '' }, finish_reason: null }] })}\n\n`,
)
response.write(
`data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: { content: mockReply }, finish_reason: null }] })}\n\n`,
)
response.write(
`data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], usage })}\n\n`,
)
response.write('data: [DONE]\n\n')
response.end()
return
}

response.writeHead(200, { 'Content-Type': 'application/json' })
response.end(
JSON.stringify({
id,
object: 'chat.completion',
created,
model: MOCK_MODEL_ID,
choices: [{ index: 0, message: { role: 'assistant', content: mockReply }, finish_reason: 'stop' }],
usage,
}),
)
return
}

response.writeHead(404, { 'Content-Type': 'application/json' })
response.end(JSON.stringify({ error: { message: `unhandled ${request.method} ${url}` } }))
})
}

function git(directory: string, ...args: string[]): void {
execFileSync('git', args, { cwd: directory, stdio: 'pipe' })
}

async function createSession(api: APIRequestContext, directory: string): Promise<string> {
const response = await api.post('/api/opencode/api/session', { data: { location: { directory } } })
if (!response.ok()) {
throw new Error(`create session failed: ${response.status()} ${await response.text()}`)
}
const body = (await response.json()) as { data: { id: string } }
return body.data.id
}

test.beforeAll(async () => {
const server = http.createServer(handleMockRequest)
await new Promise<void>((resolve, reject) => {
server.once('error', reject)
server.listen(MOCK_PROVIDER_PORT, '127.0.0.1', () => resolve())
mockProvider = await startMockProvider(MOCK_PROVIDER_PORT)
mockRepo = await createMockRepo('e2e-inline-mock-repo', MOCK_PROVIDER_PORT, {
beforeCommit: (directory) => {
fs.mkdirSync(path.join(directory, 'docs', 'img'), { recursive: true })
fs.writeFileSync(path.join(directory, 'docs', 'img', 'logo.png'), PNG_BUFFER)
fs.writeFileSync(path.join(directory, 'docs', 'notes.txt'), 'attachment notes\n')
},
})
mockServer = server

const directory = path.join(E2E_TMP, 'workspace', 'repos', 'e2e-inline-mock-repo')
fs.rmSync(directory, { recursive: true, force: true })
fs.mkdirSync(path.join(directory, 'docs', 'img'), { recursive: true })
fs.writeFileSync(path.join(directory, 'docs', 'img', 'logo.png'), PNG_BUFFER)
fs.writeFileSync(path.join(directory, 'docs', 'notes.txt'), 'attachment notes\n')
fs.writeFileSync(
path.join(directory, 'opencode.json'),
JSON.stringify(
{
$schema: 'https://opencode.ai/config.json',
provider: {
mock: {
npm: '@ai-sdk/openai-compatible',
name: 'Mock',
options: { baseURL: `http://127.0.0.1:${MOCK_PROVIDER_PORT}/v1`, apiKey: 'mock' },
models: { [MOCK_MODEL_ID]: { name: 'Mock Model' } },
},
},
model: `mock/${MOCK_MODEL_ID}`,
small_model: `mock/${MOCK_MODEL_ID}`,
},
null,
2,
),
)
fs.writeFileSync(path.join(directory, 'README.md'), '# mock repository\n')
git(directory, '-c', 'init.defaultBranch=main', 'init', '-q')
git(directory, '-c', 'user.name=e2e', '-c', 'user.email=e2e@example.com', 'add', '-A')
git(directory, '-c', 'user.name=e2e', '-c', 'user.email=e2e@example.com', 'commit', '-q', '-m', 'initial commit')

const api = await playwrightRequest.newContext({ baseURL: BASE_URL, storageState: STORAGE_STATE })
try {
const repoResponse = await api.post('/api/repos', { data: { localPath: directory } })
if (!repoResponse.ok()) {
throw new Error(`register mock repo failed: ${repoResponse.status()} ${await repoResponse.text()}`)
}
const repo = (await repoResponse.json()) as { id: number; fullPath: string }
const sessionId = await createSession(api, repo.fullPath)
mockRepo = { id: repo.id, fullPath: repo.fullPath, sessionId }
} finally {
await api.dispose()
}
})

test.afterAll(async () => {
const server = mockServer
if (!server) return
await new Promise<void>((resolve) => server.close(() => resolve()))
mockServer = undefined
await mockProvider.close()
})

test.describe('inline markdown images', () => {
Expand Down Expand Up @@ -221,7 +85,7 @@ test.describe('inline markdown images', () => {
})

test('assistant markdown images resolve relative, absolute and inline data sources', async ({ page }) => {
mockReply = [
mockProvider.setReply([
'Here are the images:',
'',
'![rel](docs/img/logo.png)',
Expand All @@ -231,7 +95,7 @@ test.describe('inline markdown images', () => {
`![dataimg](data:image/png;base64,${PNG_BASE64})`,
'',
'![bad](data:text/html,hello)',
].join('\n')
].join('\n'))

await page.goto(`/repos/${mockRepo.id}/sessions/${mockRepo.sessionId}`)
const composer = page.getByPlaceholder('Send a message...')
Expand All @@ -257,7 +121,7 @@ test.describe('inline markdown images', () => {
})

test('user attachments render images as thumbnails and other files as chips', async ({ page, request }) => {
mockReply = 'Received your files.'
mockProvider.setReply('Received your files.')
const sessionId = await createSession(request, mockRepo.fullPath)

await page.goto(`/repos/${mockRepo.id}/sessions/${sessionId}`)
Expand Down
178 changes: 178 additions & 0 deletions e2e/mockProvider.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@
import { execFileSync } from 'node:child_process'
import fs from 'node:fs'
import http from 'node:http'
import path from 'node:path'
import { request as playwrightRequest, type APIRequestContext } from '@playwright/test'
import { BASE_URL, E2E_TMP, STORAGE_STATE } from './env'

const MOCK_MODEL_ID = 'mock-model'

export interface MockProvider {
setReply(text: string): void
close(): Promise<void>
}

export interface MockRepo {
id: number
fullPath: string
sessionId: string
}

export interface CreateMockRepoOptions {
beforeCommit?: (directory: string) => void
}

function handleMockRequest(
request: http.IncomingMessage,
response: http.ServerResponse,
getReply: () => string,
): void {
const chunks: Buffer[] = []
request.on('data', (chunk: Buffer) => chunks.push(chunk))
request.on('end', () => {
const url = request.url ?? ''
if (request.method === 'GET' && url.startsWith('/v1/models')) {
response.writeHead(200, { 'Content-Type': 'application/json' })
response.end(
JSON.stringify({
object: 'list',
data: [{ id: MOCK_MODEL_ID, object: 'model', created: 0, owned_by: 'mock' }],
}),
)
return
}

if (request.method === 'POST' && url.startsWith('/v1/chat/completions')) {
let body: { stream?: boolean } = {}
try {
body = JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}')
} catch {
body = {}
}

const mockReply = getReply()
const id = 'chatcmpl-mock'
const created = Math.floor(Date.now() / 1000)
const usage = { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }

if (body.stream) {
response.writeHead(200, {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
Connection: 'keep-alive',
})
const base = { id, object: 'chat.completion.chunk', created, model: MOCK_MODEL_ID }
response.write(
`data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: { role: 'assistant', content: '' }, finish_reason: null }] })}\n\n`,
)
response.write(
`data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: { content: mockReply }, finish_reason: null }] })}\n\n`,
)
response.write(
`data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], usage })}\n\n`,
)
response.write('data: [DONE]\n\n')
response.end()
return
}

response.writeHead(200, { 'Content-Type': 'application/json' })
response.end(
JSON.stringify({
id,
object: 'chat.completion',
created,
model: MOCK_MODEL_ID,
choices: [{ index: 0, message: { role: 'assistant', content: mockReply }, finish_reason: 'stop' }],
usage,
}),
)
return
}

response.writeHead(404, { 'Content-Type': 'application/json' })
response.end(JSON.stringify({ error: { message: `unhandled ${request.method} ${url}` } }))
})
}

export async function startMockProvider(port: number): Promise<MockProvider> {
let mockReply = ''
const server = http.createServer((request, response) => {
handleMockRequest(request, response, () => mockReply)
})
await new Promise<void>((resolve, reject) => {
server.once('error', reject)
server.listen(port, '127.0.0.1', () => resolve())
})
return {
setReply: (text) => {
mockReply = text
},
close: () =>
new Promise<void>((resolve) => {
server.close(() => resolve())
server.closeAllConnections()
}),
}
}

function git(directory: string, ...args: string[]): void {
execFileSync('git', args, { cwd: directory, stdio: 'pipe' })
}

export async function createSession(api: APIRequestContext, directory: string): Promise<string> {
const response = await api.post('/api/opencode/api/session', { data: { location: { directory } } })
if (!response.ok()) {
throw new Error(`create session failed: ${response.status()} ${await response.text()}`)
}
const body = (await response.json()) as { data: { id: string } }
return body.data.id
}

export async function createMockRepo(
name: string,
port: number,
options: CreateMockRepoOptions = {},
): Promise<MockRepo> {
const directory = path.join(E2E_TMP, 'workspace', 'repos', name)
fs.rmSync(directory, { recursive: true, force: true })
fs.mkdirSync(directory, { recursive: true })
fs.writeFileSync(
path.join(directory, 'opencode.json'),
JSON.stringify(
{
$schema: 'https://opencode.ai/config.json',
provider: {
mock: {
npm: '@ai-sdk/openai-compatible',
name: 'Mock',
options: { baseURL: `http://127.0.0.1:${port}/v1`, apiKey: 'mock' },
models: { [MOCK_MODEL_ID]: { name: 'Mock Model' } },
},
},
model: `mock/${MOCK_MODEL_ID}`,
small_model: `mock/${MOCK_MODEL_ID}`,
},
null,
2,
),
)
fs.writeFileSync(path.join(directory, 'README.md'), '# mock repository\n')
options.beforeCommit?.(directory)
git(directory, '-c', 'init.defaultBranch=main', 'init', '-q')
git(directory, '-c', 'user.name=e2e', '-c', 'user.email=e2e@example.com', 'add', '-A')
git(directory, '-c', 'user.name=e2e', '-c', 'user.email=e2e@example.com', 'commit', '-q', '-m', 'initial commit')

const api = await playwrightRequest.newContext({ baseURL: BASE_URL, storageState: STORAGE_STATE })
try {
const repoResponse = await api.post('/api/repos', { data: { localPath: directory } })
if (!repoResponse.ok()) {
throw new Error(`register mock repo failed: ${repoResponse.status()} ${await repoResponse.text()}`)
}
const repo = (await repoResponse.json()) as { id: number; fullPath: string }
const sessionId = await createSession(api, repo.fullPath)
return { id: repo.id, fullPath: repo.fullPath, sessionId }
} finally {
await api.dispose()
}
}
Loading