/*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ // @ts-check /** * Chat memory leak checker — state-based approach. * * The idea: if you return to the same state you started from, memory should * return to roughly the same level. Any residual growth is a potential leak. * * Each iteration: * 1. Open a fresh chat (baseline state) * 2. Measure heap + DOM nodes * 3. Cycle through ALL registered perf scenarios (text, code blocks, * tool calls, thinking, multi-turn, etc.) * 4. Open a new chat (return to baseline state — clears previous session) * 5. Measure heap + DOM nodes again * 6. The delta is the "leaked" memory for that iteration * * Multiple iterations let us detect consistent leaks vs. one-time caching. * * Usage: * npm run perf:chat-leak # defaults from config * npm run perf:chat-leak -- --iterations 5 # more iterations * npm run perf:chat-leak -- --threshold 5 # 5MB total threshold * npm run perf:chat-leak -- --build 1.115.0 # test a specific build */ const fs = require('fs'); const path = require('path'); const { DATA_DIR, loadConfig, resolveBuild, buildEnv, buildArgs, prepareRunDir, launchVSCode, } = require('./common/utils'); const { CONTENT_SCENARIOS, TOOL_CALL_SCENARIOS, MULTI_TURN_SCENARIOS, } = require('./common/perf-scenarios'); const { getUserTurns, getModelTurnCount, } = require('./common/mock-llm-server'); // -- Config (edit config.jsonc to change defaults) --------------------------- const CONFIG = loadConfig('memLeaks'); // -- CLI args ---------------------------------------------------------------- function parseArgs() { const args = process.argv.slice(2); const opts = { iterations: CONFIG.iterations ?? 3, messages: CONFIG.messages ?? 5, verbose: false, ci: false, /** @type {string | undefined} */ build: undefined, leakThresholdMB: CONFIG.leakThresholdMB ?? 5, /** @type {Record} */ settingsOverrides: {}, }; for (let i = 0; i < args.length; i++) { switch (args[i]) { case '--iterations': opts.iterations = parseInt(args[++i], 10); break; case '--messages': case '-n': opts.messages = parseInt(args[++i], 10); break; case '--verbose': opts.verbose = true; break; case '--ci': opts.ci = true; break; case '--build': case '-b': opts.build = args[++i]; break; case '--threshold': opts.leakThresholdMB = parseFloat(args[++i]); break; case '--setting': { const kv = args[++i]; const eq = kv.indexOf('='); if (eq === -1) { console.error(`--setting requires key=value, got: ${kv}`); process.exit(1); } const key = kv.slice(0, eq); const raw = kv.slice(eq + 1); const val = raw === 'true' ? true : raw === 'false' ? false : /^-?\d+(\.\d+)?$/.test(raw) ? Number(raw) : raw; opts.settingsOverrides[key] = val; break; } case '--help': case '-h': console.log([ 'Chat memory leak checker (state-based)', '', 'Options:', ' --iterations Number of open→work→reset cycles (default: 3)', ' --messages Messages to send per iteration (default: 5)', ' --ci CI mode: write Markdown summary to ci-summary.md', ' --build Path to VS Code build or version to download', ' --threshold Max total residual heap growth in MB (default: 5)', ' --setting Set a VS Code setting override (repeatable)', ' --verbose Print per-step details', ].join('\n')); process.exit(0); } } return opts; } // -- Scenario list ----------------------------------------------------------- /** * Build a flat list of scenario IDs to cycle through during leak testing. * Includes all scenario types: content-only, tool-call, and multi-turn. * * Content scenarios exercise varied rendering (code blocks, markdown, etc.). * Tool-call scenarios exercise the agent loop (model → tool → model → ...). * Multi-turn scenarios exercise user follow-ups and thinking blocks. */ function getScenarioIds() { return [ ...Object.keys(CONTENT_SCENARIOS), ...Object.keys(TOOL_CALL_SCENARIOS), ...Object.keys(MULTI_TURN_SCENARIOS), ]; } // -- Helpers ----------------------------------------------------------------- const CHAT_VIEW = 'div[id="workbench.panel.chat"]'; const CHAT_EDITOR_SEL = `${CHAT_VIEW} .interactive-input-part .monaco-editor[role="code"]`; /** * Measure heap (MB) and DOM node count after forced GC. * @param {any} cdp * @param {import('playwright').Page} page */ async function measure(cdp, page) { await cdp.send('HeapProfiler.collectGarbage'); await new Promise(r => setTimeout(r, 500)); await cdp.send('HeapProfiler.collectGarbage'); await new Promise(r => setTimeout(r, 300)); const heapInfo = /** @type {any} */ (await cdp.send('Runtime.getHeapUsage')); const heapMB = Math.round(heapInfo.usedSize / 1024 / 1024 * 100) / 100; const domNodes = await page.evaluate(() => document.querySelectorAll('*').length); return { heapMB, domNodes }; } /** * Open a new chat session via the command palette. * @param {import('playwright').Page} page */ async function openNewChat(page) { // Use keyboard shortcut to open a new chat (clears previous session) const newChatShortcut = process.platform === 'darwin' ? 'Meta+KeyL' : 'Control+KeyL'; await page.keyboard.press(newChatShortcut); await new Promise(r => setTimeout(r, 1000)); // Verify the chat view is visible and ready await page.waitForSelector(CHAT_VIEW, { timeout: 15_000 }); await page.waitForFunction( (sel) => Array.from(document.querySelectorAll(sel)).some(el => el.getBoundingClientRect().width > 0), CHAT_EDITOR_SEL, { timeout: 15_000 }, ); await new Promise(r => setTimeout(r, 500)); } /** * Send a single message and wait for the response to complete. * For multi-turn scenarios where the model makes multiple tool-call rounds * before producing content, `modelTurns` controls how many completions to * wait for. * @param {import('playwright').Page} page * @param {{ completionCount: () => number, waitForCompletion: (n: number, ms: number) => Promise }} mockServer * @param {string} text * @param {number} [modelTurns=1] - number of model completions to wait for */ async function sendMessage(page, mockServer, text, modelTurns = 1) { await page.click(CHAT_EDITOR_SEL); await new Promise(r => setTimeout(r, 200)); const inputSel = await page.evaluate((editorSel) => { const ed = document.querySelector(editorSel); if (!ed) { throw new Error('no editor'); } return ed.querySelector('.native-edit-context') ? editorSel + ' .native-edit-context' : editorSel + ' textarea'; }, CHAT_EDITOR_SEL); const hasDriver = await page.evaluate(() => // @ts-ignore !!globalThis.driver?.typeInEditor ).catch(() => false); if (hasDriver) { await page.evaluate(({ selector, t }) => { // @ts-ignore return globalThis.driver.typeInEditor(selector, t); }, { selector: inputSel, t: text }); } else { await page.click(inputSel); await new Promise(r => setTimeout(r, 200)); await page.locator(inputSel).pressSequentially(text, { delay: 0 }); } const compBefore = mockServer.completionCount(); await page.keyboard.press('Enter'); try { await mockServer.waitForCompletion(compBefore + modelTurns, 60_000); } catch { } const responseSelector = `${CHAT_VIEW} .interactive-item-container.interactive-response`; await page.waitForFunction( (sel) => { const responses = document.querySelectorAll(sel); if (responses.length === 0) { return false; } return !responses[responses.length - 1].classList.contains('chat-response-loading'); }, responseSelector, { timeout: 30_000 }, ); await new Promise(r => setTimeout(r, 500)); } /** * Run a full scenario: send the initial message, then handle any user * follow-up turns for multi-turn scenarios. * * - Content-only scenarios: single message, 1 model turn. * - Tool-call scenarios (no user turns): single message, N model turns * (the extension automatically relays tool results back to the model). * - Multi-turn with user turns: send initial message, wait for response, * then for each user turn send the follow-up message and wait again. * * @param {import('playwright').Page} page * @param {{ completionCount: () => number, waitForCompletion: (n: number, ms: number) => Promise }} mockServer * @param {string} scenarioId * @param {string} label - prefix for the message (e.g. "Warmup" or "Iteration 2") */ async function runScenario(page, mockServer, scenarioId, label) { const userTurns = getUserTurns(scenarioId); const totalModelTurns = getModelTurnCount(scenarioId); if (userTurns.length === 0) { // Content-only or tool-call scenario: one message, wait for all model turns await sendMessage(page, mockServer, `[scenario:${scenarioId}] ${label}`, totalModelTurns); } else { // Multi-turn with user follow-ups: send initial message and wait for // the model turns before the first user turn, then alternate. let modelTurnsSoFar = 0; const firstUserAfter = userTurns[0].afterModelTurn; const turnsBeforeFirstUser = firstUserAfter - modelTurnsSoFar; await sendMessage(page, mockServer, `[scenario:${scenarioId}] ${label}`, turnsBeforeFirstUser); modelTurnsSoFar = firstUserAfter; for (let u = 0; u < userTurns.length; u++) { const nextModelStop = u + 1 < userTurns.length ? userTurns[u + 1].afterModelTurn : totalModelTurns; const turnsUntilNext = nextModelStop - modelTurnsSoFar; // Send the user follow-up message await sendMessage(page, mockServer, userTurns[u].message, turnsUntilNext); modelTurnsSoFar = nextModelStop; } } } // -- Leak check -------------------------------------------------------------- /** * @param {string} electronPath * @param {{ url: string, requestCount: () => number, waitForRequests: (n: number, ms: number) => Promise, completionCount: () => number, waitForCompletion: (n: number, ms: number) => Promise }} mockServer * @param {{ iterations: number, verbose: boolean, settingsOverrides?: Record }} opts */ async function runLeakCheck(electronPath, mockServer, opts) { const { iterations, verbose } = opts; const { userDataDir, extDir, logsDir } = prepareRunDir('leak-check', mockServer, opts.settingsOverrides); const isDevBuild = !electronPath.includes('.vscode-test'); const vscode = await launchVSCode( electronPath, buildArgs(userDataDir, extDir, logsDir, { isDevBuild }), buildEnv(mockServer, { isDevBuild }), { verbose }, ); const page = vscode.page; try { await page.waitForSelector('.monaco-workbench', { timeout: 60_000 }); const cdp = await page.context().newCDPSession(page); await cdp.send('HeapProfiler.enable'); // Open chat panel const chatShortcut = process.platform === 'darwin' ? 'Control+Meta+KeyI' : 'Control+Alt+KeyI'; await page.keyboard.press(chatShortcut); await page.waitForSelector(CHAT_VIEW, { timeout: 15_000 }); await page.waitForFunction( (sel) => Array.from(document.querySelectorAll(sel)).some(el => el.getBoundingClientRect().width > 0), CHAT_EDITOR_SEL, { timeout: 15_000 }, ); // Wait for extension activation const reqsBefore = mockServer.requestCount(); try { await mockServer.waitForRequests(reqsBefore + 4, 30_000); } catch { } await new Promise(r => setTimeout(r, 3000)); const scenarioIds = getScenarioIds(); // --- Baseline measurement (fresh chat) --- const baseline = await measure(cdp, page); if (verbose) { console.log(` [leak] Baseline: heap=${baseline.heapMB}MB, domNodes=${baseline.domNodes}`); } /** @type {{ beforeHeapMB: number, afterHeapMB: number, deltaHeapMB: number, beforeDomNodes: number, afterDomNodes: number, deltaDomNodes: number }[]} */ const iterationResults = []; for (let iter = 0; iter < iterations; iter++) { // Measure at start of iteration (should be in "clean" state) const before = await measure(cdp, page); if (verbose) { console.log(` [leak] Iteration ${iter + 1}/${iterations}: start heap=${before.heapMB}MB, domNodes=${before.domNodes}`); } // Do work: cycle through all scenarios for (let m = 0; m < scenarioIds.length; m++) { const sid = scenarioIds[m]; await runScenario(page, mockServer, sid, `Iteration ${iter + 1}`); if (verbose) { console.log(` [leak] Sent ${sid} (${m + 1}/${scenarioIds.length})`); } } // Return to clean state: open a new empty chat await openNewChat(page); await new Promise(r => setTimeout(r, 1000)); // Measure after returning to clean state const after = await measure(cdp, page); const deltaHeapMB = Math.round((after.heapMB - before.heapMB) * 100) / 100; const deltaDomNodes = after.domNodes - before.domNodes; iterationResults.push({ beforeHeapMB: before.heapMB, afterHeapMB: after.heapMB, deltaHeapMB, beforeDomNodes: before.domNodes, afterDomNodes: after.domNodes, deltaDomNodes, }); if (verbose) { console.log(` [leak] Iteration ${iter + 1}/${iterations}: end heap=${after.heapMB}MB (delta=${deltaHeapMB}MB), domNodes=${after.domNodes} (delta=${deltaDomNodes})`); } } // Final measurement const final = await measure(cdp, page); const totalResidualMB = Math.round((final.heapMB - baseline.heapMB) * 100) / 100; const totalResidualNodes = final.domNodes - baseline.domNodes; return { baseline, final: { heapMB: final.heapMB, domNodes: final.domNodes }, totalResidualMB, totalResidualNodes, iterations: iterationResults, }; } finally { await vscode.close(); } } // -- Main -------------------------------------------------------------------- async function main() { const opts = parseArgs(); const electronPath = await resolveBuild(opts.build); if (!fs.existsSync(electronPath)) { console.error(`Electron not found at: ${electronPath}`); process.exit(1); } const { startServer } = require('./common/mock-llm-server'); const { registerPerfScenarios } = require('./common/perf-scenarios'); registerPerfScenarios(); const mockServer = await startServer(0); console.log(`[chat-simulation] Leak check: ${opts.iterations} iterations × ${getScenarioIds().length} scenarios, threshold ${opts.leakThresholdMB}MB total`); console.log(`[chat-simulation] Build: ${electronPath}`); console.log(''); const result = await runLeakCheck(electronPath, mockServer, opts); console.log('[chat-simulation] =================== Leak Check Results ==================='); console.log(''); console.log(` Baseline: heap=${result.baseline.heapMB}MB, domNodes=${result.baseline.domNodes}`); console.log(` Final: heap=${result.final.heapMB}MB, domNodes=${result.final.domNodes}`); console.log(''); for (let i = 0; i < result.iterations.length; i++) { const it = result.iterations[i]; console.log(` Iteration ${i + 1}: ${it.beforeHeapMB}MB → ${it.afterHeapMB}MB (residual: ${it.deltaHeapMB > 0 ? '+' : ''}${it.deltaHeapMB}MB, DOM: ${it.deltaDomNodes > 0 ? '+' : ''}${it.deltaDomNodes} nodes)`); } console.log(''); console.log(` Total residual heap growth: ${result.totalResidualMB > 0 ? '+' : ''}${result.totalResidualMB}MB`); console.log(` Total residual DOM growth: ${result.totalResidualNodes > 0 ? '+' : ''}${result.totalResidualNodes} nodes`); console.log(''); // Write JSON const jsonPath = path.join(DATA_DIR, 'chat-simulation-leak-results.json'); fs.writeFileSync(jsonPath, JSON.stringify({ timestamp: new Date().toISOString(), leakThresholdMB: opts.leakThresholdMB, iterationCount: opts.iterations, scenarioCount: getScenarioIds().length, ...result, }, null, 2)); console.log(`[chat-simulation] Results written to ${jsonPath}`); const leaked = result.totalResidualMB > opts.leakThresholdMB; console.log(''); if (leaked) { console.log(`[chat-simulation] LEAK DETECTED — ${result.totalResidualMB}MB residual exceeds ${opts.leakThresholdMB}MB threshold`); } else { console.log(`[chat-simulation] No leak detected (${result.totalResidualMB}MB residual < ${opts.leakThresholdMB}MB threshold)`); } if (opts.ci) { const summary = generateLeakCISummary(result, opts); const summaryPath = path.join(DATA_DIR, 'ci-summary-leak.md'); fs.writeFileSync(summaryPath, summary); console.log(`[chat-simulation] CI summary written to ${summaryPath}`); } await mockServer.close(); process.exit(leaked ? 1 : 0); } /** * Generate a Markdown summary for CI, matching the perf script pattern. * @param {{ baseline: { heapMB: number, domNodes: number }, final: { heapMB: number, domNodes: number }, totalResidualMB: number, totalResidualNodes: number, iterations: { beforeHeapMB: number, afterHeapMB: number, deltaHeapMB: number, beforeDomNodes: number, afterDomNodes: number, deltaDomNodes: number }[] }} result * @param {{ leakThresholdMB: number, iterations: number }} opts */ function generateLeakCISummary(result, opts) { const leaked = result.totalResidualMB > opts.leakThresholdMB; const verdict = leaked ? '\u274C **LEAK DETECTED**' : '\u2705 **No leak detected**'; const lines = []; lines.push('## Memory Leak Check'); lines.push(''); lines.push('| | |'); lines.push('|---|---|'); lines.push(`| **Verdict** | ${verdict} |`); lines.push(`| **Threshold** | ${opts.leakThresholdMB} MB |`); lines.push(`| **Iterations** | ${opts.iterations} |`); lines.push(`| **Scenarios per iteration** | ${getScenarioIds().length} |`); lines.push(''); lines.push('| Phase | Heap (MB) | DOM Nodes |'); lines.push('|-------|----------:|----------:|'); lines.push(`| Baseline | ${result.baseline.heapMB} | ${result.baseline.domNodes} |`); for (let i = 0; i < result.iterations.length; i++) { const it = result.iterations[i]; const sign = it.deltaHeapMB > 0 ? '+' : ''; const domSign = it.deltaDomNodes > 0 ? '+' : ''; lines.push(`| Iteration ${i + 1} | ${it.afterHeapMB} (${sign}${it.deltaHeapMB}) | ${it.afterDomNodes} (${domSign}${it.deltaDomNodes}) |`); } lines.push(`| **Final** | **${result.final.heapMB}** | **${result.final.domNodes}** |`); lines.push(''); const sign = result.totalResidualMB > 0 ? '+' : ''; const domSign = result.totalResidualNodes > 0 ? '+' : ''; lines.push(`**Total residual growth:** ${sign}${result.totalResidualMB} MB heap, ${domSign}${result.totalResidualNodes} DOM nodes`); lines.push(''); return lines.join('\n'); } main().catch(err => { console.error(err); process.exit(1); });