diff --git a/apps/cli/ai/mcp-server.ts b/apps/cli/ai/mcp-server.ts index 4f2a7a0ae2..46cfe26163 100644 --- a/apps/cli/ai/mcp-server.ts +++ b/apps/cli/ai/mcp-server.ts @@ -1,8 +1,13 @@ import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; import { createStudioTools } from 'cli/ai/tools'; +import { type McpTelemetryGroup } from 'cli/lib/types/bump-stats'; -export async function startMcpStdioServer(): Promise< void > { - const studioMcp = createStudioTools(); +type McpServerOptions = { + telemetryGroup?: McpTelemetryGroup; +}; + +export async function startMcpStdioServer( options: McpServerOptions = {} ): Promise< void > { + const studioMcp = createStudioTools( options ); const transport = new StdioServerTransport(); const shutdown = async () => { diff --git a/apps/cli/ai/tests/tools.test.ts b/apps/cli/ai/tests/tools.test.ts index cc48dc8a7f..a2380cafb7 100644 --- a/apps/cli/ai/tests/tools.test.ts +++ b/apps/cli/ai/tests/tools.test.ts @@ -6,11 +6,12 @@ import { } from 'cli/commands/preview/delete'; import { runCommand as runListPreviewCommand } from 'cli/commands/preview/list'; import { runCommand as runUpdatePreviewCommand } from 'cli/commands/preview/update'; +import { bumpStat } from 'cli/lib/bump-stat'; import { readCliConfig } from 'cli/lib/cli-config/core'; import { getSiteByFolder } from 'cli/lib/cli-config/sites'; import { isServerRunning, sendWpCliCommand } from 'cli/lib/wordpress-server-manager'; import { getProgressCallback, setProgressCallback } from 'cli/logger'; -import { studioToolDefinitions } from '../tools'; +import { createStudioToolDefinitions, studioToolDefinitions } from '../tools'; vi.mock( 'cli/ai/block-validator', () => ( { validateBlocks: vi.fn(), @@ -74,6 +75,12 @@ vi.mock( 'cli/lib/cli-config/sites', async () => ( { getSiteByFolder: vi.fn(), } ) ); +vi.mock( 'cli/lib/bump-stat', () => ( { + bumpStat: vi.fn(), + bumpAggregatedUniqueStat: vi.fn(), + getPlatformMetric: vi.fn(), +} ) ); + vi.mock( 'cli/lib/daemon-client', () => ( { connectToDaemon: vi.fn(), disconnectFromDaemon: vi.fn(), @@ -130,6 +137,30 @@ describe( 'Studio AI MCP tools', () => { ); } ); + it( 'only includes workflow telemetry when a plugin group is configured', () => { + expect( studioToolDefinitions.map( ( tool ) => tool.name ) ).not.toContain( + 'record_workflow_event' + ); + expect( createStudioToolDefinitions( 'codex-plugin' ).map( ( tool ) => tool.name ) ).toContain( + 'record_workflow_event' + ); + } ); + + it( 'records workflow telemetry for a configured plugin group', async () => { + const tool = createStudioToolDefinitions( 'codex-plugin' ).find( + ( definition ) => definition.name === 'record_workflow_event' + ); + expect( tool ).toBeDefined(); + + const result = await tool!.handler( + { workflow: 'theme-build', stage: 'started' } as never, + null + ); + + expect( bumpStat ).toHaveBeenCalledWith( 'codex-plugin', 'theme-build-started' ); + expect( getTextContent( result ) ).toContain( 'codex-plugin theme-build-started' ); + } ); + it( 'creates previews for a resolved local site', async () => { const result = await getTool( 'preview_create' ).handler( { nameOrPath: 'My Site' } as never, diff --git a/apps/cli/ai/tools.ts b/apps/cli/ai/tools.ts index 656376091c..e76fe6db78 100644 --- a/apps/cli/ai/tools.ts +++ b/apps/cli/ai/tools.ts @@ -19,11 +19,18 @@ import { runCommand as runListSitesCommand } from 'cli/commands/site/list'; import { runCommand as runStartSiteCommand } from 'cli/commands/site/start'; import { runCommand as runStatusCommand } from 'cli/commands/site/status'; import { runCommand as runStopSiteCommand, Mode as StopMode } from 'cli/commands/site/stop'; +import { bumpStat } from 'cli/lib/bump-stat'; import { readCliConfig, type SiteData } from 'cli/lib/cli-config/core'; import { getSiteByFolder, getSiteUrl } from 'cli/lib/cli-config/sites'; import { connectToDaemon, disconnectFromDaemon } from 'cli/lib/daemon-client'; import { getUnsupportedWpCliPostContentMessage } from 'cli/lib/rewrite-wp-cli-post-content'; import { STUDIO_SITES_ROOT } from 'cli/lib/site-paths'; +import { + MCP_WORKFLOWS, + MCP_WORKFLOW_STAGES, + type McpTelemetryGroup, + type McpWorkflowStat, +} from 'cli/lib/types/bump-stats'; import { normalizeHostname } from 'cli/lib/utils'; import { isServerRunning, sendWpCliCommand } from 'cli/lib/wordpress-server-manager'; import { getProgressCallback, setProgressCallback, emitProgress } from 'cli/logger'; @@ -128,6 +135,33 @@ function textResult( text: string ) { }; } +function createRecordWorkflowEventTool( telemetryGroup?: McpTelemetryGroup ) { + return tool( + 'record_workflow_event', + 'Records a workflow telemetry event for the active AI plugin integration, such as theme-build started or completed.', + { + workflow: z + .enum( MCP_WORKFLOWS ) + .describe( 'Workflow name, such as theme-build or site-build' ), + stage: z + .enum( MCP_WORKFLOW_STAGES ) + .describe( 'Workflow stage to record: started, completed, or failed' ), + }, + async ( args ) => { + const stat = `${ args.workflow }-${ args.stage }` as McpWorkflowStat; + + if ( ! telemetryGroup ) { + return textResult( + `Workflow telemetry skipped for ${ stat }: no plugin group configured.` + ); + } + + bumpStat( telemetryGroup, stat ); + return textResult( `Workflow telemetry recorded: ${ telemetryGroup } ${ stat }.` ); + } + ); +} + function formatInvalidBlocks( report: ValidationReport ): string[] { const lines: string[] = []; for ( const result of report.results ) { @@ -761,28 +795,38 @@ const auditPerformanceTool = tool( } ); -export const studioToolDefinitions = [ - createSiteTool, - listSitesTool, - getSiteInfoTool, - startSiteTool, - stopSiteTool, - deleteSiteTool, - createPreviewTool, - listPreviewsTool, - updatePreviewTool, - deletePreviewTool, - runWpCliTool, - validateBlocksTool, - takeScreenshotTool, - installTaxonomyScriptsTool, - auditPerformanceTool, -]; - -export function createStudioTools() { +export function createStudioToolDefinitions( telemetryGroup?: McpTelemetryGroup ) { + const tools = [ + createSiteTool, + listSitesTool, + getSiteInfoTool, + startSiteTool, + stopSiteTool, + deleteSiteTool, + createPreviewTool, + listPreviewsTool, + updatePreviewTool, + deletePreviewTool, + runWpCliTool, + validateBlocksTool, + takeScreenshotTool, + installTaxonomyScriptsTool, + auditPerformanceTool, + ]; + + if ( telemetryGroup ) { + return [ ...tools, createRecordWorkflowEventTool( telemetryGroup ) ]; + } + + return tools; +} + +export const studioToolDefinitions = createStudioToolDefinitions(); + +export function createStudioTools( options: { telemetryGroup?: McpTelemetryGroup } = {} ) { return createSdkMcpServer( { name: 'studio', version: '1.0.0', - tools: studioToolDefinitions, + tools: createStudioToolDefinitions( options.telemetryGroup ), } ); } diff --git a/apps/cli/ai/ui.ts b/apps/cli/ai/ui.ts index 5717d406f4..34ac69b0e7 100644 --- a/apps/cli/ai/ui.ts +++ b/apps/cli/ai/ui.ts @@ -243,6 +243,7 @@ const toolDisplayNames: Record< string, string > = { mcp__studio__wp_cli: __( 'Run WP-CLI' ), mcp__studio__validate_blocks: __( 'Validate blocks' ), mcp__studio__take_screenshot: __( 'Take screenshot' ), + mcp__studio__record_workflow_event: __( 'Record workflow telemetry' ), Read: __( 'Read' ), Write: __( 'Write' ), Edit: __( 'Edit' ), @@ -277,6 +278,11 @@ function getToolDetail( name: string, input: Record< string, unknown > ): string return __( 'inline content' ); case 'mcp__studio__take_screenshot': return typeof input.url === 'string' ? input.url : ''; + case 'mcp__studio__record_workflow_event': + if ( typeof input.workflow === 'string' && typeof input.stage === 'string' ) { + return `${ input.workflow } ${ input.stage }`; + } + return ''; case 'Read': case 'Write': case 'Edit': { diff --git a/apps/cli/commands/mcp.ts b/apps/cli/commands/mcp.ts index d829a0fbdb..e5cd66202a 100644 --- a/apps/cli/commands/mcp.ts +++ b/apps/cli/commands/mcp.ts @@ -5,6 +5,7 @@ import { } from '@studio/common/lib/mcp-config'; import { __ } from '@wordpress/i18n'; import { startMcpStdioServer } from 'cli/ai/mcp-server'; +import { MCP_TELEMETRY_GROUPS, type McpTelemetryGroup } from 'cli/lib/types/bump-stats'; import { Logger, LoggerError } from 'cli/logger'; import { StudioArgv } from 'cli/types'; @@ -63,13 +64,17 @@ function printInstallationInstructions(): void { } } -export async function runCommand(): Promise< void > { +type McpCommandOptions = { + telemetryGroup?: McpTelemetryGroup; +}; + +export async function runCommand( options: McpCommandOptions = {} ): Promise< void > { if ( process.stdin.isTTY && process.stdout.isTTY ) { printInstallationInstructions(); return; } - await startMcpStdioServer(); + await startMcpStdioServer( options ); } export const registerCommand = ( yargs: StudioArgv ) => { @@ -77,7 +82,14 @@ export const registerCommand = ( yargs: StudioArgv ) => { command: 'mcp', describe: __( 'MCP server for AI assistants' ), builder: ( yargs ) => { - return yargs.help( false ).option( 'help', { type: 'boolean' } ); + return yargs + .help( false ) + .option( 'help', { type: 'boolean' } ) + .option( 'telemetry-group', { + type: 'string', + hidden: true, + choices: [ ...MCP_TELEMETRY_GROUPS ], + } ); }, handler: async ( argv ) => { if ( argv.help ) { @@ -85,7 +97,9 @@ export const registerCommand = ( yargs: StudioArgv ) => { return; } try { - await runCommand(); + await runCommand( { + telemetryGroup: argv.telemetryGroup as McpTelemetryGroup | undefined, + } ); } catch ( error ) { if ( error instanceof LoggerError ) { logger.reportError( error ); diff --git a/apps/cli/lib/bump-stat.ts b/apps/cli/lib/bump-stat.ts index f04b0a604a..2ae54c4b25 100644 --- a/apps/cli/lib/bump-stat.ts +++ b/apps/cli/lib/bump-stat.ts @@ -10,7 +10,12 @@ import { saveCliConfig, unlockCliConfig, } from 'cli/lib/cli-config/core'; -import { StatsGroup, StatsMetric } from 'cli/lib/types/bump-stats'; +import { + type BumpStatGroup, + type BumpStatMetric, + StatsGroup, + StatsMetric, +} from 'cli/lib/types/bump-stats'; const lastBumpStatsProvider: LastBumpStatsProvider = { load: async () => { @@ -28,7 +33,7 @@ const lastBumpStatsProvider: LastBumpStatsProvider = { }, }; -export function bumpStat( group: StatsGroup, stat: StatsMetric, bumpInDev = false ) { +export function bumpStat( group: BumpStatGroup, stat: BumpStatMetric, bumpInDev = false ) { return __bumpStat( group, stat, bumpInDev ); } diff --git a/apps/cli/lib/types/bump-stats.ts b/apps/cli/lib/types/bump-stats.ts index b59e12da3a..28748ca364 100644 --- a/apps/cli/lib/types/bump-stats.ts +++ b/apps/cli/lib/types/bump-stats.ts @@ -4,6 +4,25 @@ export enum StatsGroup { STUDIO_CLI_WEEKLY_UNIQUE_APP = 'studio-cli-weekly-unq-app', } +export const MCP_TELEMETRY_GROUPS = [ 'claude-code-plugin', 'codex-plugin' ] as const; +export type McpTelemetryGroup = ( typeof MCP_TELEMETRY_GROUPS )[ number ]; + +export const MCP_WORKFLOWS = [ + 'site-build', + 'theme-build', + 'block-build', + 'plugin-build', + 'auditing', +] as const; +export type McpWorkflow = ( typeof MCP_WORKFLOWS )[ number ]; + +export const MCP_WORKFLOW_STAGES = [ 'started', 'completed', 'failed' ] as const; +export type McpWorkflowStage = ( typeof MCP_WORKFLOW_STAGES )[ number ]; +export type McpWorkflowStat = `${ McpWorkflow }-${ McpWorkflowStage }`; + +export type BumpStatGroup = StatsGroup | McpTelemetryGroup; +export type BumpStatMetric = StatsMetric | McpWorkflowStat; + export enum StatsMetric { SUCCESS = 'success', FAILURE = 'failure',