From b0d2abe4218eef4f9bbfdf504e8c04d71dcff4ec Mon Sep 17 00:00:00 2001 From: Mihir-Mavalankar Date: Wed, 8 Apr 2026 11:53:55 -0700 Subject: [PATCH 1/3] fix(seer): Replace internal Unicode operators with readable labels in LLM context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Widget conditions use \uf00d-delimited operators (e.g. Containsqueue.task) that are invisible but garble the Seer agent's understanding, causing broken tool calls. Replace these with readable labels (e.g. " contains ") via simple string replacement that preserves all query structure — AND, OR, parens, free text, and comparison operators pass through unchanged. Co-Authored-By: Claude Opus 4.6 --- .../app/views/dashboards/widgetCard/index.tsx | 4 +- .../widgetCard/widgetLLMContext.spec.tsx | 124 +++++++++++++++++- .../widgetCard/widgetLLMContext.tsx | 17 +++ 3 files changed, 142 insertions(+), 3 deletions(-) diff --git a/static/app/views/dashboards/widgetCard/index.tsx b/static/app/views/dashboards/widgetCard/index.tsx index b9b7d1a3f5637c..167f259dd866f7 100644 --- a/static/app/views/dashboards/widgetCard/index.tsx +++ b/static/app/views/dashboards/widgetCard/index.tsx @@ -64,7 +64,7 @@ import { useTransactionsDeprecationWarning, } from './widgetCardContextMenu'; import {WidgetFrame} from './widgetFrame'; -import {getWidgetQueryLLMHint} from './widgetLLMContext'; +import {getWidgetQueryLLMHint, readableConditions} from './widgetLLMContext'; export type OnDataFetchedParams = { tableResults?: TableDataWithTitle[]; @@ -165,7 +165,7 @@ function WidgetCard(props: Props) { queryHint: getWidgetQueryLLMHint(resolvedDisplayType), queries: props.widget.queries.map(q => ({ name: q.name, - conditions: q.conditions, + conditions: readableConditions(q.conditions), aggregates: q.aggregates, columns: q.columns, orderby: q.orderby, diff --git a/static/app/views/dashboards/widgetCard/widgetLLMContext.spec.tsx b/static/app/views/dashboards/widgetCard/widgetLLMContext.spec.tsx index ef531f3e34b1ab..5681cd866a3f1d 100644 --- a/static/app/views/dashboards/widgetCard/widgetLLMContext.spec.tsx +++ b/static/app/views/dashboards/widgetCard/widgetLLMContext.spec.tsx @@ -1,6 +1,6 @@ import {DisplayType} from 'sentry/views/dashboards/types'; -import {getWidgetQueryLLMHint} from './widgetLLMContext'; +import {getWidgetQueryLLMHint, readableConditions} from './widgetLLMContext'; describe('getWidgetQueryLLMHint', () => { it.each([ @@ -26,3 +26,125 @@ describe('getWidgetQueryLLMHint', () => { expect(getWidgetQueryLLMHint(DisplayType.WHEEL)).toContain('table query'); }); }); + +describe('readableConditions', () => { + it('replaces Contains operator with readable label', () => { + expect(readableConditions('span.name:\uf00dContains\uf00dfoo')).toBe( + 'span.name: contains foo' + ); + }); + + it('replaces Contains with IN list brackets', () => { + expect(readableConditions('span.name:\uf00dContains\uf00d[a,b,c]')).toBe( + 'span.name: contains [a,b,c]' + ); + }); + + it('replaces DoesNotContain operator', () => { + expect(readableConditions('key:\uf00dDoesNotContain\uf00dval')).toBe( + 'key: does not contain val' + ); + }); + + it('replaces StartsWith and EndsWith operators', () => { + expect(readableConditions('key:\uf00dStartsWith\uf00d/api')).toBe( + 'key: starts with /api' + ); + expect(readableConditions('key:\uf00dEndsWith\uf00d.json')).toBe( + 'key: ends with .json' + ); + }); + + it('replaces DoesNotStartWith and DoesNotEndWith operators', () => { + expect(readableConditions('key:\uf00dDoesNotStartWith\uf00d/api')).toBe( + 'key: does not start with /api' + ); + expect(readableConditions('key:\uf00dDoesNotEndWith\uf00d.json')).toBe( + 'key: does not end with .json' + ); + }); + + it('preserves negated filter prefix', () => { + expect(readableConditions('!path:\uf00dContains\uf00dfoo')).toBe( + '!path: contains foo' + ); + }); + + it('replaces multiple operators in one string', () => { + const input = + 'span.name:\uf00dContains\uf00dqueue.task !trigger_path:\uf00dContains\uf00dold_seer'; + expect(readableConditions(input)).toBe( + 'span.name: contains queue.task !trigger_path: contains old_seer' + ); + }); + + it('passes through plain filters unchanged', () => { + expect(readableConditions('browser.name:Firefox')).toBe('browser.name:Firefox'); + }); + + it('passes through free text unchanged', () => { + expect(readableConditions('some free text')).toBe('some free text'); + }); + + it('passes through empty string', () => { + expect(readableConditions('')).toBe(''); + }); + + it('preserves OR and parentheses', () => { + expect(readableConditions('(a:1 OR b:2) error')).toBe('(a:1 OR b:2) error'); + }); + + it('preserves comparison operators', () => { + expect(readableConditions('count():>100 duration:<=5s')).toBe( + 'count():>100 duration:<=5s' + ); + }); + + it('handles real-world widget query', () => { + const input = + 'span.description:\uf00dContains\uf00d[sentry.tasks.autofix.generate_issue_summary_only,sentry.tasks.autofix.run_automation_only_task] span.name:\uf00dContains\uf00dqueue.task.taskworker !trigger_path:\uf00dContains\uf00dold_seer_automation'; + expect(readableConditions(input)).toBe( + 'span.description: contains [sentry.tasks.autofix.generate_issue_summary_only,sentry.tasks.autofix.run_automation_only_task] span.name: contains queue.task.taskworker !trigger_path: contains old_seer_automation' + ); + }); + + it('does not replace DoesNotContain partially as Contains', () => { + // DoesNotContain must be replaced before Contains to avoid partial match + expect(readableConditions('key:\uf00dDoesNotContain\uf00dval')).toBe( + 'key: does not contain val' + ); + // Should NOT produce "key: does not contains val" or "key:DoesNot contains val" + expect(readableConditions('key:\uf00dDoesNotContain\uf00dval')).not.toContain( + '\uf00d' + ); + }); + + it('handles mixed operator types in one query', () => { + const input = + 'url:\uf00dStartsWith\uf00d/api span.description:\uf00dContains\uf00dfoo !path:\uf00dDoesNotEndWith\uf00d.js'; + expect(readableConditions(input)).toBe( + 'url: starts with /api span.description: contains foo !path: does not end with .js' + ); + }); + + it('handles the same operator appearing multiple times', () => { + const input = + 'a:\uf00dContains\uf00dfoo b:\uf00dContains\uf00dbar c:\uf00dContains\uf00dbaz'; + expect(readableConditions(input)).toBe( + 'a: contains foo b: contains bar c: contains baz' + ); + }); + + it('preserves OR with wildcard operators inside parens', () => { + const input = + '(span.name:\uf00dContains\uf00dfoo OR span.name:\uf00dContains\uf00dbar)'; + expect(readableConditions(input)).toBe( + '(span.name: contains foo OR span.name: contains bar)' + ); + }); + + it('does not replace literal "Contains" text without unicode markers', () => { + // The word "Contains" in a value or free text should NOT be replaced + expect(readableConditions('message:Contains error')).toBe('message:Contains error'); + }); +}); diff --git a/static/app/views/dashboards/widgetCard/widgetLLMContext.tsx b/static/app/views/dashboards/widgetCard/widgetLLMContext.tsx index c792a7ed1b34cd..91b2cf76abfd92 100644 --- a/static/app/views/dashboards/widgetCard/widgetLLMContext.tsx +++ b/static/app/views/dashboards/widgetCard/widgetLLMContext.tsx @@ -1,5 +1,22 @@ import {DisplayType} from 'sentry/views/dashboards/types'; +/** + * Replace internal \uf00d-delimited wildcard operators with readable labels + * so the Seer Explorer agent can understand widget filter conditions. + * + * All other query structure (AND, OR, parens, free text, comparison operators) + * passes through unchanged since only wildcard operators use \uf00d markers. + */ +export function readableConditions(query: string): string { + return query + .replaceAll('\uf00dDoesNotContain\uf00d', ' does not contain ') + .replaceAll('\uf00dDoesNotStartWith\uf00d', ' does not start with ') + .replaceAll('\uf00dDoesNotEndWith\uf00d', ' does not end with ') + .replaceAll('\uf00dContains\uf00d', ' contains ') + .replaceAll('\uf00dStartsWith\uf00d', ' starts with ') + .replaceAll('\uf00dEndsWith\uf00d', ' ends with '); +} + /** * Returns a hint for the Seer Explorer agent describing how to re-query this * widget's data using a tool call, if the user wants to dig deeper. From 39ea4be42e1f1081ea7fdbd2e18ac5fc74f2172a Mon Sep 17 00:00:00 2001 From: Mihir-Mavalankar Date: Wed, 8 Apr 2026 12:18:58 -0700 Subject: [PATCH 2/3] ref(seer): Improve widget query hints for natural language agent The Seer agent uses natural language tool calls, not raw API params. Replace hints that reference internal params (y_axes, group_by, query) with descriptions of what the widget visualizes and direct the agent to understand the intent from the query config. Co-Authored-By: Claude Opus 4.6 --- static/app/views/dashboards/dashboard.tsx | 2 +- .../widgetCard/widgetLLMContext.spec.tsx | 18 +++++++++--------- .../dashboards/widgetCard/widgetLLMContext.tsx | 12 ++++++------ 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/static/app/views/dashboards/dashboard.tsx b/static/app/views/dashboards/dashboard.tsx index b4500bba4baffd..02972f837385e1 100644 --- a/static/app/views/dashboards/dashboard.tsx +++ b/static/app/views/dashboards/dashboard.tsx @@ -129,7 +129,7 @@ function DashboardInner({ // Push dashboard metadata into the LLM context tree for Seer Explorer. useLLMContext({ contextHint: - 'This is a Sentry dashboard. The dateRange, environments, and projects below are global page filters that scope every widget query. Each child widget node contains its own query config that can be used with the telemetry_live_search tool to fetch or drill into its data.', + 'This is a Sentry dashboard. The dateRange, environments, and projects below are global page filters that scope every widget query. Each child widget node contains its own query config that can be used with tools like telemetry_live_search and telemetry_index_list_nodes to fetch data for that widget and dig deeper. Based on the user question, data might be needed from multiple widgets.', title: dashboard.title, widgetCount: dashboard.widgets.length, filters: dashboard.filters, diff --git a/static/app/views/dashboards/widgetCard/widgetLLMContext.spec.tsx b/static/app/views/dashboards/widgetCard/widgetLLMContext.spec.tsx index 5681cd866a3f1d..0a35e2a12dace7 100644 --- a/static/app/views/dashboards/widgetCard/widgetLLMContext.spec.tsx +++ b/static/app/views/dashboards/widgetCard/widgetLLMContext.spec.tsx @@ -4,26 +4,26 @@ import {getWidgetQueryLLMHint, readableConditions} from './widgetLLMContext'; describe('getWidgetQueryLLMHint', () => { it.each([ - [DisplayType.LINE, 'timeseries'], - [DisplayType.AREA, 'timeseries'], - [DisplayType.BAR, 'timeseries'], + [DisplayType.LINE, 'timeseries chart'], + [DisplayType.AREA, 'timeseries chart'], + [DisplayType.BAR, 'timeseries chart'], ])('returns timeseries hint for %s', (displayType, expected) => { expect(getWidgetQueryLLMHint(displayType)).toContain(expected); }); it('returns table hint for TABLE', () => { - expect(getWidgetQueryLLMHint(DisplayType.TABLE)).toContain('table query'); + expect(getWidgetQueryLLMHint(DisplayType.TABLE)).toContain('shows a table'); }); - it('returns single aggregate hint for BIG_NUMBER', () => { - expect(getWidgetQueryLLMHint(DisplayType.BIG_NUMBER)).toContain('single aggregate'); + it('returns single number hint for BIG_NUMBER', () => { + expect(getWidgetQueryLLMHint(DisplayType.BIG_NUMBER)).toContain('single number'); expect(getWidgetQueryLLMHint(DisplayType.BIG_NUMBER)).toContain( - 'value is included below' + 'current value is included below' ); }); - it('returns table hint as default for unknown types', () => { - expect(getWidgetQueryLLMHint(DisplayType.WHEEL)).toContain('table query'); + it('returns generic hint for unknown types', () => { + expect(getWidgetQueryLLMHint(DisplayType.WHEEL)).toContain('shows data'); }); }); diff --git a/static/app/views/dashboards/widgetCard/widgetLLMContext.tsx b/static/app/views/dashboards/widgetCard/widgetLLMContext.tsx index 91b2cf76abfd92..a0c6c8565cb783 100644 --- a/static/app/views/dashboards/widgetCard/widgetLLMContext.tsx +++ b/static/app/views/dashboards/widgetCard/widgetLLMContext.tsx @@ -18,20 +18,20 @@ export function readableConditions(query: string): string { } /** - * Returns a hint for the Seer Explorer agent describing how to re-query this - * widget's data using a tool call, if the user wants to dig deeper. + * Returns a hint for the Seer Explorer agent describing what this widget + * visualizes so it can understand the intent from the query config. */ export function getWidgetQueryLLMHint(displayType: DisplayType): string { switch (displayType) { case DisplayType.LINE: case DisplayType.AREA: case DisplayType.BAR: - return 'To dig deeper into this widget, run a timeseries query using y_axes (aggregates) + group_by (columns) + query (conditions)'; + return 'This widget shows a timeseries chart. The aggregates are the y-axis metrics, columns are the group-by breakdowns, and conditions filter the data. Understand the intent from the query config below.'; case DisplayType.TABLE: - return 'To dig deeper into this widget, run a table query using fields (aggregates + columns) + query (conditions) + sort (orderby)'; + return 'This widget shows a table. The aggregates and columns define the visible fields, orderby is the sort, and conditions filter the data. Understand the intent from the query config below.'; case DisplayType.BIG_NUMBER: - return 'To dig deeper into this widget, run a single aggregate query using fields (aggregates) + query (conditions); current value is included below'; + return 'This widget shows a single number. The aggregate is the metric, conditions filter the data, and the current value is included below. Understand the intent from the query config below.'; default: - return 'To dig deeper into this widget, run a table query using fields (aggregates + columns) + query (conditions)'; + return 'This widget shows data. The aggregates, columns, and conditions define what is displayed. Understand the intent from the query config below.'; } } From 898ff84eca8c5d8f12678d8bc5489ed4c4571c62 Mon Sep 17 00:00:00 2001 From: Mihir-Mavalankar Date: Wed, 8 Apr 2026 13:26:47 -0700 Subject: [PATCH 3/3] ref(seer): Use OP_LABELS to dynamically replace wildcard operators Instead of hardcoding 6 replaceAll calls, derive replacements from OP_LABELS by filtering to keys containing \uf00d markers. This automatically handles any new operators added to the map. Co-Authored-By: Claude Opus 4.6 --- .../views/dashboards/widgetCard/widgetLLMContext.tsx | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/static/app/views/dashboards/widgetCard/widgetLLMContext.tsx b/static/app/views/dashboards/widgetCard/widgetLLMContext.tsx index a0c6c8565cb783..f8f0343263764a 100644 --- a/static/app/views/dashboards/widgetCard/widgetLLMContext.tsx +++ b/static/app/views/dashboards/widgetCard/widgetLLMContext.tsx @@ -1,3 +1,4 @@ +import {OP_LABELS} from 'sentry/components/searchQueryBuilder/tokens/filter/utils'; import {DisplayType} from 'sentry/views/dashboards/types'; /** @@ -8,13 +9,9 @@ import {DisplayType} from 'sentry/views/dashboards/types'; * passes through unchanged since only wildcard operators use \uf00d markers. */ export function readableConditions(query: string): string { - return query - .replaceAll('\uf00dDoesNotContain\uf00d', ' does not contain ') - .replaceAll('\uf00dDoesNotStartWith\uf00d', ' does not start with ') - .replaceAll('\uf00dDoesNotEndWith\uf00d', ' does not end with ') - .replaceAll('\uf00dContains\uf00d', ' contains ') - .replaceAll('\uf00dStartsWith\uf00d', ' starts with ') - .replaceAll('\uf00dEndsWith\uf00d', ' ends with '); + return Object.entries(OP_LABELS) + .filter(([key]) => key.includes('\uf00d')) + .reduce((s, [key, label]) => s.replaceAll(key, ` ${label} `), query); } /**