From d181cf87d15d421b04bc2aede661c50126c90acf Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 27 Mar 2026 10:03:56 +0000 Subject: [PATCH 01/12] Initial plan From 9a72c0f780025bc526e3402fca2d7e727d351629 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 27 Mar 2026 10:14:40 +0000 Subject: [PATCH 02/12] Add system(command; args) operator with --enable-system-operator flag Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/8a11e9a0-10d2-4f2a-ae29-4e9d0bfc266f Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> --- cmd/root.go | 1 + .../doc/operators/headers/system-operators.md | 23 ++++ pkg/yqlib/doc/operators/system-operators.md | 72 +++++++++++++ pkg/yqlib/lexer_participle.go | 2 + pkg/yqlib/operation.go | 2 + pkg/yqlib/operator_system.go | 102 ++++++++++++++++++ pkg/yqlib/operator_system_test.go | 84 +++++++++++++++ pkg/yqlib/security_prefs.go | 10 +- 8 files changed, 292 insertions(+), 4 deletions(-) create mode 100644 pkg/yqlib/doc/operators/headers/system-operators.md create mode 100644 pkg/yqlib/doc/operators/system-operators.md create mode 100644 pkg/yqlib/operator_system.go create mode 100644 pkg/yqlib/operator_system_test.go diff --git a/cmd/root.go b/cmd/root.go index 449be02518..18960b5165 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -212,6 +212,7 @@ yq -P -oy sample.json rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableEnvOps, "security-disable-env-ops", "", false, "Disable env related operations.") rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableFileOps, "security-disable-file-ops", "", false, "Disable file related operations (e.g. load)") + rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.EnableSystemOps, "enable-system-operator", "", false, "Enable system operator to allow execution of external commands.") rootCmd.AddCommand( createEvaluateSequenceCommand(), diff --git a/pkg/yqlib/doc/operators/headers/system-operators.md b/pkg/yqlib/doc/operators/headers/system-operators.md new file mode 100644 index 0000000000..badb4c61eb --- /dev/null +++ b/pkg/yqlib/doc/operators/headers/system-operators.md @@ -0,0 +1,23 @@ +# System Operators + +The `system` operator allows you to run an external command and use its output as a value in your expression. + +**Security warning**: The system operator is disabled by default. You must explicitly pass `--enable-system-operator` to use it. + +## Usage + +```bash +yq --enable-system-operator '.field = system("command"; "arg1")' +``` + +The operator takes: +- A command string (required) +- An argument or array of arguments separated by `;` (optional) + +The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string. + +## Disabling the system operator + +The system operator is disabled by default. When disabled, a warning is logged and `null` is returned instead of running the command. + +Use `--enable-system-operator` flag to enable it. diff --git a/pkg/yqlib/doc/operators/system-operators.md b/pkg/yqlib/doc/operators/system-operators.md new file mode 100644 index 0000000000..a29acadc5e --- /dev/null +++ b/pkg/yqlib/doc/operators/system-operators.md @@ -0,0 +1,72 @@ +# System Operators + +The `system` operator allows you to run an external command and use its output as a value in your expression. + +**Security warning**: The system operator is disabled by default. You must explicitly pass `--enable-system-operator` to use it. + +## Usage + +```bash +yq --enable-system-operator '.field = system("command"; "arg1")' +``` + +The operator takes: +- A command string (required) +- An argument or array of arguments separated by `;` (optional) + +The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string. + +## Disabling the system operator + +The system operator is disabled by default. When disabled, a warning is logged and `null` is returned instead of running the command. + +Use `--enable-system-operator` flag to enable it. + +## system operator returns null when disabled +Use `--enable-system-operator` to enable the system operator. + +Given a sample.yml file of: +```yaml +country: Australia +``` +then +```bash +yq '.country = system("/usr/bin/echo"; "test")' sample.yml +``` +will output +```yaml +country: null +``` + +## Run a command with an argument +Use `--enable-system-operator` to enable the system operator. + +Given a sample.yml file of: +```yaml +country: Australia +``` +then +```bash +yq '.country = system("/usr/bin/echo"; "test")' sample.yml +``` +will output +```yaml +country: test +``` + +## Run a command without arguments +Omit the semicolon and args to run the command with no extra arguments. + +Given a sample.yml file of: +```yaml +a: hello +``` +then +```bash +yq '.a = system("/bin/echo")' sample.yml +``` +will output +```yaml +a: "" +``` + diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go index 866b736cb4..938b8a717a 100644 --- a/pkg/yqlib/lexer_participle.go +++ b/pkg/yqlib/lexer_participle.go @@ -96,6 +96,8 @@ var participleYqRules = []*participleYqRule{ simpleOp("load_?str|str_?load", loadStringOpType), {"LoadYaml", `load`, loadOp(NewYamlDecoder(LoadYamlPreferences)), 0}, + simpleOp("system", systemOpType), + {"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0}, simpleOp("select", selectOpType), diff --git a/pkg/yqlib/operation.go b/pkg/yqlib/operation.go index f36054eec9..cbea1d7b9c 100644 --- a/pkg/yqlib/operation.go +++ b/pkg/yqlib/operation.go @@ -164,6 +164,8 @@ var stringInterpolationOpType = &operationType{Type: "STRING_INT", NumArgs: 0, P var loadOpType = &operationType{Type: "LOAD", NumArgs: 1, Precedence: 52, Handler: loadOperator, CheckForPostTraverse: true} var loadStringOpType = &operationType{Type: "LOAD_STRING", NumArgs: 1, Precedence: 52, Handler: loadStringOperator} +var systemOpType = &operationType{Type: "SYSTEM", NumArgs: 1, Precedence: 50, Handler: systemOperator} + var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 52, Handler: keysOperator, CheckForPostTraverse: true} var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator} diff --git a/pkg/yqlib/operator_system.go b/pkg/yqlib/operator_system.go new file mode 100644 index 0000000000..e2d71203c6 --- /dev/null +++ b/pkg/yqlib/operator_system.go @@ -0,0 +1,102 @@ +package yqlib + +import ( + "bytes" + "container/list" + "fmt" + "os/exec" + "strings" +) + +func systemOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { + if !ConfiguredSecurityPreferences.EnableSystemOps { + log.Warning("system operator is disabled, use --enable-system-operator flag to enable") + results := list.New() + for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + results.PushBack(candidate.CreateReplacement(ScalarNode, "!!null", "null")) + } + return context.ChildContext(results), nil + } + + var command string + var argsExpression *ExpressionNode + + // check if it's a block operator (command; args) or just (command) + if expressionNode.RHS.Operation.OperationType == blockOpType { + block := expressionNode.RHS + commandNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), block.LHS) + if err != nil { + return Context{}, err + } + if commandNodes.MatchingNodes.Front() == nil { + return Context{}, fmt.Errorf("system operator: command expression returned no results") + } + command = commandNodes.MatchingNodes.Front().Value.(*CandidateNode).Value + argsExpression = block.RHS + } else { + commandNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) + if err != nil { + return Context{}, err + } + if commandNodes.MatchingNodes.Front() == nil { + return Context{}, fmt.Errorf("system operator: command expression returned no results") + } + command = commandNodes.MatchingNodes.Front().Value.(*CandidateNode).Value + } + + // evaluate args if present + var args []string + if argsExpression != nil { + argsNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), argsExpression) + if err != nil { + return Context{}, err + } + if argsNodes.MatchingNodes.Front() != nil { + argsNode := argsNodes.MatchingNodes.Front().Value.(*CandidateNode) + if argsNode.Kind == SequenceNode { + for _, child := range argsNode.Content { + args = append(args, child.Value) + } + } else if argsNode.Tag != "!!null" { + args = []string{argsNode.Value} + } + } + } + + var results = list.New() + + for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + + var stdin bytes.Buffer + if candidate.Tag != "!!null" { + encoded, err := encodeToYamlString(candidate) + if err != nil { + return Context{}, err + } + stdin.WriteString(encoded) + } + + // #nosec G204 - intentional: user must explicitly enable this operator + cmd := exec.Command(command, args...) + cmd.Stdin = &stdin + var stderr bytes.Buffer + cmd.Stderr = &stderr + + output, err := cmd.Output() + if err != nil { + stderrStr := strings.TrimSpace(stderr.String()) + if stderrStr != "" { + return Context{}, fmt.Errorf("system command '%v' failed: %w\nstderr: %v", command, err, stderrStr) + } + return Context{}, fmt.Errorf("system command '%v' failed: %w", command, err) + } + + result := strings.TrimRight(string(output), "\n") + newNode := candidate.CreateReplacement(ScalarNode, "!!str", result) + results.PushBack(newNode) + } + + return context.ChildContext(results), nil +} diff --git a/pkg/yqlib/operator_system_test.go b/pkg/yqlib/operator_system_test.go new file mode 100644 index 0000000000..64f1a3f397 --- /dev/null +++ b/pkg/yqlib/operator_system_test.go @@ -0,0 +1,84 @@ +package yqlib + +import ( + "testing" +) + +var systemOperatorDisabledScenarios = []expressionScenario{ + { + description: "system operator returns null when disabled", + subdescription: "Use `--enable-system-operator` to enable the system operator.", + document: "country: Australia", + expression: `.country = system("/usr/bin/echo"; "test")`, + expected: []string{ + "D0, P[], (!!map)::country: null\n", + }, + }, +} + +var systemOperatorEnabledScenarios = []expressionScenario{ + { + description: "Run a command with an argument", + subdescription: "Use `--enable-system-operator` to enable the system operator.", + document: "country: Australia", + expression: `.country = system("/usr/bin/echo"; "test")`, + expected: []string{ + "D0, P[], (!!map)::country: test\n", + }, + }, + { + description: "Run a command without arguments", + subdescription: "Omit the semicolon and args to run the command with no extra arguments.", + document: "a: hello", + expression: `.a = system("/bin/echo")`, + expected: []string{ + "D0, P[], (!!map)::a: \"\"\n", + }, + }, + { + description: "Run a command with multiple arguments", + subdescription: "Pass an array of arguments.", + skipDoc: true, + document: "a: hello", + expression: `.a = system("/bin/echo"; ["foo", "bar"])`, + expected: []string{ + "D0, P[], (!!map)::a: foo bar\n", + }, + }, + { + description: "Command failure returns error", + skipDoc: true, + document: "a: hello", + expression: `.a = system("/bin/false")`, + expectedError: "system command '/bin/false' failed: exit status 1", + }, +} + +func TestSystemOperatorDisabledScenarios(t *testing.T) { + // ensure system operator is disabled + originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps + defer func() { + ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps + }() + + ConfiguredSecurityPreferences.EnableSystemOps = false + + for _, tt := range systemOperatorDisabledScenarios { + testScenario(t, &tt) + } + documentOperatorScenarios(t, "system-operators", systemOperatorDisabledScenarios) +} + +func TestSystemOperatorEnabledScenarios(t *testing.T) { + originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps + defer func() { + ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps + }() + + ConfiguredSecurityPreferences.EnableSystemOps = true + + for _, tt := range systemOperatorEnabledScenarios { + testScenario(t, &tt) + } + appendOperatorDocumentScenario(t, "system-operators", systemOperatorEnabledScenarios) +} diff --git a/pkg/yqlib/security_prefs.go b/pkg/yqlib/security_prefs.go index 3e2fe6b49a..3c203014f2 100644 --- a/pkg/yqlib/security_prefs.go +++ b/pkg/yqlib/security_prefs.go @@ -1,11 +1,13 @@ package yqlib type SecurityPreferences struct { - DisableEnvOps bool - DisableFileOps bool + DisableEnvOps bool + DisableFileOps bool + EnableSystemOps bool } var ConfiguredSecurityPreferences = SecurityPreferences{ - DisableEnvOps: false, - DisableFileOps: false, + DisableEnvOps: false, + DisableFileOps: false, + EnableSystemOps: false, } From 6315cfe974c7e9390ccee2f85da98a55b51bce0c Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sat, 28 Mar 2026 12:59:37 +1100 Subject: [PATCH 03/12] Update pkg/yqlib/operator_system.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- pkg/yqlib/operator_system.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/yqlib/operator_system.go b/pkg/yqlib/operator_system.go index e2d71203c6..f717393938 100644 --- a/pkg/yqlib/operator_system.go +++ b/pkg/yqlib/operator_system.go @@ -93,7 +93,12 @@ func systemOperator(d *dataTreeNavigator, context Context, expressionNode *Expre return Context{}, fmt.Errorf("system command '%v' failed: %w", command, err) } - result := strings.TrimRight(string(output), "\n") + result := string(output) + if strings.HasSuffix(result, "\r\n") { + result = result[:len(result)-2] + } else if strings.HasSuffix(result, "\n") { + result = result[:len(result)-1] + } newNode := candidate.CreateReplacement(ScalarNode, "!!str", result) results.PushBack(newNode) } From da611f7a2b051ed2f25c3cd62e79d78909b3410b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 28 Mar 2026 02:03:15 +0000 Subject: [PATCH 04/12] Evaluate system command/args per matched node using SingleReadonlyChildContext Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/dca841eb-3f63-4f23-adeb-556431560420 Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> --- pkg/yqlib/doc/operators/system-operators.md | 2 +- pkg/yqlib/operator_system.go | 93 +++++++++--------- pkg/yqlib/operator_system_test.go | 103 ++++++++++++-------- 3 files changed, 112 insertions(+), 86 deletions(-) diff --git a/pkg/yqlib/doc/operators/system-operators.md b/pkg/yqlib/doc/operators/system-operators.md index a29acadc5e..f9c0346ca4 100644 --- a/pkg/yqlib/doc/operators/system-operators.md +++ b/pkg/yqlib/doc/operators/system-operators.md @@ -63,7 +63,7 @@ a: hello ``` then ```bash -yq '.a = system("/bin/echo")' sample.yml +yq '.a = system("/usr/bin/echo")' sample.yml ``` will output ```yaml diff --git a/pkg/yqlib/operator_system.go b/pkg/yqlib/operator_system.go index f717393938..f2e8c9b772 100644 --- a/pkg/yqlib/operator_system.go +++ b/pkg/yqlib/operator_system.go @@ -8,6 +8,20 @@ import ( "strings" ) +func resolveSystemArgs(argsNode *CandidateNode) []string { + if argsNode.Kind == SequenceNode { + args := make([]string, 0, len(argsNode.Content)) + for _, child := range argsNode.Content { + args = append(args, child.Value) + } + return args + } + if argsNode.Tag != "!!null" { + return []string{argsNode.Value} + } + return nil +} + func systemOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { if !ConfiguredSecurityPreferences.EnableSystemOps { log.Warning("system operator is disabled, use --enable-system-operator flag to enable") @@ -19,55 +33,46 @@ func systemOperator(d *dataTreeNavigator, context Context, expressionNode *Expre return context.ChildContext(results), nil } - var command string - var argsExpression *ExpressionNode - - // check if it's a block operator (command; args) or just (command) - if expressionNode.RHS.Operation.OperationType == blockOpType { - block := expressionNode.RHS - commandNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), block.LHS) - if err != nil { - return Context{}, err - } - if commandNodes.MatchingNodes.Front() == nil { - return Context{}, fmt.Errorf("system operator: command expression returned no results") - } - command = commandNodes.MatchingNodes.Front().Value.(*CandidateNode).Value - argsExpression = block.RHS - } else { - commandNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) - if err != nil { - return Context{}, err - } - if commandNodes.MatchingNodes.Front() == nil { - return Context{}, fmt.Errorf("system operator: command expression returned no results") - } - command = commandNodes.MatchingNodes.Front().Value.(*CandidateNode).Value - } - - // evaluate args if present - var args []string - if argsExpression != nil { - argsNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), argsExpression) - if err != nil { - return Context{}, err - } - if argsNodes.MatchingNodes.Front() != nil { - argsNode := argsNodes.MatchingNodes.Front().Value.(*CandidateNode) - if argsNode.Kind == SequenceNode { - for _, child := range argsNode.Content { - args = append(args, child.Value) - } - } else if argsNode.Tag != "!!null" { - args = []string{argsNode.Value} - } - } - } + // determine at parse time whether we have (command; args) or just (command) + hasArgs := expressionNode.RHS.Operation.OperationType == blockOpType var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) + nodeContext := context.SingleReadonlyChildContext(candidate) + + var command string + var args []string + + if hasArgs { + block := expressionNode.RHS + commandNodes, err := d.GetMatchingNodes(nodeContext, block.LHS) + if err != nil { + return Context{}, err + } + if commandNodes.MatchingNodes.Front() == nil { + return Context{}, fmt.Errorf("system operator: command expression returned no results") + } + command = commandNodes.MatchingNodes.Front().Value.(*CandidateNode).Value + + argsNodes, err := d.GetMatchingNodes(nodeContext, block.RHS) + if err != nil { + return Context{}, err + } + if argsNodes.MatchingNodes.Front() != nil { + args = resolveSystemArgs(argsNodes.MatchingNodes.Front().Value.(*CandidateNode)) + } + } else { + commandNodes, err := d.GetMatchingNodes(nodeContext, expressionNode.RHS) + if err != nil { + return Context{}, err + } + if commandNodes.MatchingNodes.Front() == nil { + return Context{}, fmt.Errorf("system operator: command expression returned no results") + } + command = commandNodes.MatchingNodes.Front().Value.(*CandidateNode).Value + } var stdin bytes.Buffer if candidate.Tag != "!!null" { diff --git a/pkg/yqlib/operator_system_test.go b/pkg/yqlib/operator_system_test.go index 64f1a3f397..388f4446f8 100644 --- a/pkg/yqlib/operator_system_test.go +++ b/pkg/yqlib/operator_system_test.go @@ -1,9 +1,19 @@ package yqlib import ( + "os/exec" "testing" ) +func findExec(t *testing.T, name string) string { + t.Helper() + path, err := exec.LookPath(name) + if err != nil { + t.Skipf("skipping: %v not found: %v", name, err) + } + return path +} + var systemOperatorDisabledScenarios = []expressionScenario{ { description: "system operator returns null when disabled", @@ -16,46 +26,7 @@ var systemOperatorDisabledScenarios = []expressionScenario{ }, } -var systemOperatorEnabledScenarios = []expressionScenario{ - { - description: "Run a command with an argument", - subdescription: "Use `--enable-system-operator` to enable the system operator.", - document: "country: Australia", - expression: `.country = system("/usr/bin/echo"; "test")`, - expected: []string{ - "D0, P[], (!!map)::country: test\n", - }, - }, - { - description: "Run a command without arguments", - subdescription: "Omit the semicolon and args to run the command with no extra arguments.", - document: "a: hello", - expression: `.a = system("/bin/echo")`, - expected: []string{ - "D0, P[], (!!map)::a: \"\"\n", - }, - }, - { - description: "Run a command with multiple arguments", - subdescription: "Pass an array of arguments.", - skipDoc: true, - document: "a: hello", - expression: `.a = system("/bin/echo"; ["foo", "bar"])`, - expected: []string{ - "D0, P[], (!!map)::a: foo bar\n", - }, - }, - { - description: "Command failure returns error", - skipDoc: true, - document: "a: hello", - expression: `.a = system("/bin/false")`, - expectedError: "system command '/bin/false' failed: exit status 1", - }, -} - func TestSystemOperatorDisabledScenarios(t *testing.T) { - // ensure system operator is disabled originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps defer func() { ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps @@ -70,6 +41,9 @@ func TestSystemOperatorDisabledScenarios(t *testing.T) { } func TestSystemOperatorEnabledScenarios(t *testing.T) { + echoPath := findExec(t, "echo") + falsePath := findExec(t, "false") + originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps defer func() { ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps @@ -77,8 +51,55 @@ func TestSystemOperatorEnabledScenarios(t *testing.T) { ConfiguredSecurityPreferences.EnableSystemOps = true - for _, tt := range systemOperatorEnabledScenarios { + scenarios := []expressionScenario{ + { + description: "Run a command with an argument", + subdescription: "Use `--enable-system-operator` to enable the system operator.", + document: "country: Australia", + expression: `.country = system("` + echoPath + `"; "test")`, + expected: []string{ + "D0, P[], (!!map)::country: test\n", + }, + }, + { + description: "Run a command without arguments", + subdescription: "Omit the semicolon and args to run the command with no extra arguments.", + document: "a: hello", + expression: `.a = system("` + echoPath + `")`, + expected: []string{ + "D0, P[], (!!map)::a: \"\"\n", + }, + }, + { + description: "Run a command with multiple arguments", + subdescription: "Pass an array of arguments.", + skipDoc: true, + document: "a: hello", + expression: `.a = system("` + echoPath + `"; ["foo", "bar"])`, + expected: []string{ + "D0, P[], (!!map)::a: foo bar\n", + }, + }, + { + description: "Command and args are evaluated per matched node", + skipDoc: true, + document: "cmd: " + echoPath + "\narg: hello", + expression: `.result = system(.cmd; .arg)`, + expected: []string{ + "D0, P[], (!!map)::cmd: " + echoPath + "\narg: hello\nresult: hello\n", + }, + }, + { + description: "Command failure returns error", + skipDoc: true, + document: "a: hello", + expression: `.a = system("` + falsePath + `")`, + expectedError: "system command '" + falsePath + "' failed: exit status 1", + }, + } + + for _, tt := range scenarios { testScenario(t, &tt) } - appendOperatorDocumentScenario(t, "system-operators", systemOperatorEnabledScenarios) + appendOperatorDocumentScenario(t, "system-operators", scenarios) } From 884c2d8b6b7dd8c8ff4b0ad1caf429ed1234540a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 28 Mar 2026 02:08:14 +0000 Subject: [PATCH 05/12] Add yqFlags to expressionScenario for doc command snippets; fix system op docs Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/3f8a5375-25fd-4428-a8e6-b630194c36b2 Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> --- pkg/yqlib/doc/operators/system-operators.md | 4 ++-- pkg/yqlib/operator_system_test.go | 2 ++ pkg/yqlib/operators_test.go | 15 ++++++++++++--- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/pkg/yqlib/doc/operators/system-operators.md b/pkg/yqlib/doc/operators/system-operators.md index f9c0346ca4..0223acaae7 100644 --- a/pkg/yqlib/doc/operators/system-operators.md +++ b/pkg/yqlib/doc/operators/system-operators.md @@ -47,7 +47,7 @@ country: Australia ``` then ```bash -yq '.country = system("/usr/bin/echo"; "test")' sample.yml +yq --enable-system-operator '.country = system("/usr/bin/echo"; "test")' sample.yml ``` will output ```yaml @@ -63,7 +63,7 @@ a: hello ``` then ```bash -yq '.a = system("/usr/bin/echo")' sample.yml +yq --enable-system-operator '.a = system("/usr/bin/echo")' sample.yml ``` will output ```yaml diff --git a/pkg/yqlib/operator_system_test.go b/pkg/yqlib/operator_system_test.go index 388f4446f8..385e1561ce 100644 --- a/pkg/yqlib/operator_system_test.go +++ b/pkg/yqlib/operator_system_test.go @@ -55,6 +55,7 @@ func TestSystemOperatorEnabledScenarios(t *testing.T) { { description: "Run a command with an argument", subdescription: "Use `--enable-system-operator` to enable the system operator.", + yqFlags: "--enable-system-operator", document: "country: Australia", expression: `.country = system("` + echoPath + `"; "test")`, expected: []string{ @@ -64,6 +65,7 @@ func TestSystemOperatorEnabledScenarios(t *testing.T) { { description: "Run a command without arguments", subdescription: "Omit the semicolon and args to run the command with no extra arguments.", + yqFlags: "--enable-system-operator", document: "a: hello", expression: `.a = system("` + echoPath + `")`, expected: []string{ diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index 1646fb9b4a..b4d4301e5a 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -31,6 +31,7 @@ type expressionScenario struct { dontFormatInputForDoc bool // dont format input doc for documentation generation requiresFormat string skipForGoccy bool + yqFlags string // extra yq flags to include in generated doc command snippets } var goccyTesting = false @@ -356,14 +357,22 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) { writeOrPanic(w, "then\n") + flagsPrefix := "" + if s.yqFlags != "" { + flagsPrefix = s.yqFlags + " " + } if s.expression != "" { - writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v'%v' %v\n```\n", envCommand, command, strings.ReplaceAll(s.expression, "'", `'\''`), files)) + writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v'%v' %v\n```\n", envCommand, flagsPrefix, command, strings.ReplaceAll(s.expression, "'", `'\''`), files)) } else { - writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v\n```\n", envCommand, command, files)) + writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v%v\n```\n", envCommand, flagsPrefix, command, files)) } } else { writeOrPanic(w, "Running\n") - writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v--null-input '%v'\n```\n", envCommand, command, s.expression)) + flagsPrefix := "" + if s.yqFlags != "" { + flagsPrefix = s.yqFlags + " " + } + writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v--null-input '%v'\n```\n", envCommand, flagsPrefix, command, s.expression)) } return formattedDoc, formattedDoc2 } From 2c8605f63425230d39d1f868a07a01a73ea8dc83 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sat, 28 Mar 2026 14:21:28 +1100 Subject: [PATCH 06/12] Update pkg/yqlib/doc/operators/headers/system-operators.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- pkg/yqlib/doc/operators/headers/system-operators.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/yqlib/doc/operators/headers/system-operators.md b/pkg/yqlib/doc/operators/headers/system-operators.md index badb4c61eb..2df8fdaa7f 100644 --- a/pkg/yqlib/doc/operators/headers/system-operators.md +++ b/pkg/yqlib/doc/operators/headers/system-operators.md @@ -7,7 +7,7 @@ The `system` operator allows you to run an external command and use its output a ## Usage ```bash -yq --enable-system-operator '.field = system("command"; "arg1")' +yq --null-input --enable-system-operator '.field = system("command"; "arg1")' ``` The operator takes: From 5ea069a5ede7faf012fd71a6672171057e78b829 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sat, 28 Mar 2026 14:21:39 +1100 Subject: [PATCH 07/12] Update pkg/yqlib/doc/operators/system-operators.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- pkg/yqlib/doc/operators/system-operators.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/yqlib/doc/operators/system-operators.md b/pkg/yqlib/doc/operators/system-operators.md index 0223acaae7..765ae940f5 100644 --- a/pkg/yqlib/doc/operators/system-operators.md +++ b/pkg/yqlib/doc/operators/system-operators.md @@ -7,7 +7,7 @@ The `system` operator allows you to run an external command and use its output a ## Usage ```bash -yq --enable-system-operator '.field = system("command"; "arg1")' +yq --enable-system-operator --null-input '.field = system("command"; "arg1")' ``` The operator takes: From 53abbbaee95f63330419411474ba8e8b6d558ccc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 28 Mar 2026 03:27:46 +0000 Subject: [PATCH 08/12] Validate command node type and handle multiple results with debug log Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/928aabc5-ad71-41d8-94ab-403942e3f92d Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> --- .../doc/operators/headers/system-operators.md | 2 +- pkg/yqlib/operator_system.go | 26 ++++++++++++++----- pkg/yqlib/operator_system_test.go | 7 +++++ 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/pkg/yqlib/doc/operators/headers/system-operators.md b/pkg/yqlib/doc/operators/headers/system-operators.md index 2df8fdaa7f..9b84e8df45 100644 --- a/pkg/yqlib/doc/operators/headers/system-operators.md +++ b/pkg/yqlib/doc/operators/headers/system-operators.md @@ -7,7 +7,7 @@ The `system` operator allows you to run an external command and use its output a ## Usage ```bash -yq --null-input --enable-system-operator '.field = system("command"; "arg1")' +yq --enable-system-operator --null-input '.field = system("command"; "arg1")' ``` The operator takes: diff --git a/pkg/yqlib/operator_system.go b/pkg/yqlib/operator_system.go index f2e8c9b772..2fb5886959 100644 --- a/pkg/yqlib/operator_system.go +++ b/pkg/yqlib/operator_system.go @@ -22,6 +22,20 @@ func resolveSystemArgs(argsNode *CandidateNode) []string { return nil } +func resolveCommandNode(commandNodes Context) (string, error) { + if commandNodes.MatchingNodes.Front() == nil { + return "", fmt.Errorf("system operator: command expression returned no results") + } + if commandNodes.MatchingNodes.Len() > 1 { + log.Debugf("system operator: command expression returned %d results, using first", commandNodes.MatchingNodes.Len()) + } + cmdNode := commandNodes.MatchingNodes.Front().Value.(*CandidateNode) + if cmdNode.Kind != ScalarNode || cmdNode.Tag == "!!null" { + return "", fmt.Errorf("system operator: command must be a string scalar") + } + return cmdNode.Value, nil +} + func systemOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { if !ConfiguredSecurityPreferences.EnableSystemOps { log.Warning("system operator is disabled, use --enable-system-operator flag to enable") @@ -51,10 +65,10 @@ func systemOperator(d *dataTreeNavigator, context Context, expressionNode *Expre if err != nil { return Context{}, err } - if commandNodes.MatchingNodes.Front() == nil { - return Context{}, fmt.Errorf("system operator: command expression returned no results") + command, err = resolveCommandNode(commandNodes) + if err != nil { + return Context{}, err } - command = commandNodes.MatchingNodes.Front().Value.(*CandidateNode).Value argsNodes, err := d.GetMatchingNodes(nodeContext, block.RHS) if err != nil { @@ -68,10 +82,10 @@ func systemOperator(d *dataTreeNavigator, context Context, expressionNode *Expre if err != nil { return Context{}, err } - if commandNodes.MatchingNodes.Front() == nil { - return Context{}, fmt.Errorf("system operator: command expression returned no results") + command, err = resolveCommandNode(commandNodes) + if err != nil { + return Context{}, err } - command = commandNodes.MatchingNodes.Front().Value.(*CandidateNode).Value } var stdin bytes.Buffer diff --git a/pkg/yqlib/operator_system_test.go b/pkg/yqlib/operator_system_test.go index 385e1561ce..867e1921c3 100644 --- a/pkg/yqlib/operator_system_test.go +++ b/pkg/yqlib/operator_system_test.go @@ -98,6 +98,13 @@ func TestSystemOperatorEnabledScenarios(t *testing.T) { expression: `.a = system("` + falsePath + `")`, expectedError: "system command '" + falsePath + "' failed: exit status 1", }, + { + description: "Null command returns error", + skipDoc: true, + document: "a: hello", + expression: `.a = system(null)`, + expectedError: "system operator: command must be a string scalar", + }, } for _, tt := range scenarios { From e10e8127e1cb4986691c70aa7c7164e7cc75b81e Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sun, 29 Mar 2026 09:45:08 +1100 Subject: [PATCH 09/12] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- cmd/root.go | 6 +++++- pkg/yqlib/operator_system.go | 13 +++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 18960b5165..3dfe18e603 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -212,7 +212,11 @@ yq -P -oy sample.json rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableEnvOps, "security-disable-env-ops", "", false, "Disable env related operations.") rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableFileOps, "security-disable-file-ops", "", false, "Disable file related operations (e.g. load)") - rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.EnableSystemOps, "enable-system-operator", "", false, "Enable system operator to allow execution of external commands.") + rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.EnableSystemOps, "security-enable-system-operator", "", false, "Enable system operator to allow execution of external commands.") + rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.EnableSystemOps, "enable-system-operator", "", false, "DEPRECATED: use --security-enable-system-operator instead. Enable system operator to allow execution of external commands.") + if err = rootCmd.MarkPersistentFlagDeprecated("enable-system-operator", "use --security-enable-system-operator instead."); err != nil { + panic(err) + } rootCmd.AddCommand( createEvaluateSequenceCommand(), diff --git a/pkg/yqlib/operator_system.go b/pkg/yqlib/operator_system.go index 2fb5886959..8a37d274c6 100644 --- a/pkg/yqlib/operator_system.go +++ b/pkg/yqlib/operator_system.go @@ -33,6 +33,9 @@ func resolveCommandNode(commandNodes Context) (string, error) { if cmdNode.Kind != ScalarNode || cmdNode.Tag == "!!null" { return "", fmt.Errorf("system operator: command must be a string scalar") } + if cmdNode.Value == "" { + return "", fmt.Errorf("system operator: command must be a non-empty string") + } return cmdNode.Value, nil } @@ -89,13 +92,11 @@ func systemOperator(d *dataTreeNavigator, context Context, expressionNode *Expre } var stdin bytes.Buffer - if candidate.Tag != "!!null" { - encoded, err := encodeToYamlString(candidate) - if err != nil { - return Context{}, err - } - stdin.WriteString(encoded) + encoded, err := encodeToYamlString(candidate) + if err != nil { + return Context{}, err } + stdin.WriteString(encoded) // #nosec G204 - intentional: user must explicitly enable this operator cmd := exec.Command(command, args...) From b3b447883932732eabc9ca85a428165438ee7dc6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 6 Apr 2026 08:35:36 +0000 Subject: [PATCH 10/12] Remove deprecated --enable-system-operator alias; use --security-enable-system-operator consistently Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/286b95e9-b6d7-4ab8-b401-2d7a03853922 Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> --- cmd/root.go | 4 ---- .../doc/operators/headers/system-operators.md | 6 +++--- pkg/yqlib/doc/operators/system-operators.md | 14 +++++++------- pkg/yqlib/operator_system.go | 2 +- pkg/yqlib/operator_system_test.go | 8 ++++---- 5 files changed, 15 insertions(+), 19 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 3dfe18e603..d2f72bfa0b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -213,10 +213,6 @@ yq -P -oy sample.json rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableEnvOps, "security-disable-env-ops", "", false, "Disable env related operations.") rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableFileOps, "security-disable-file-ops", "", false, "Disable file related operations (e.g. load)") rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.EnableSystemOps, "security-enable-system-operator", "", false, "Enable system operator to allow execution of external commands.") - rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.EnableSystemOps, "enable-system-operator", "", false, "DEPRECATED: use --security-enable-system-operator instead. Enable system operator to allow execution of external commands.") - if err = rootCmd.MarkPersistentFlagDeprecated("enable-system-operator", "use --security-enable-system-operator instead."); err != nil { - panic(err) - } rootCmd.AddCommand( createEvaluateSequenceCommand(), diff --git a/pkg/yqlib/doc/operators/headers/system-operators.md b/pkg/yqlib/doc/operators/headers/system-operators.md index 9b84e8df45..a58fa7f30d 100644 --- a/pkg/yqlib/doc/operators/headers/system-operators.md +++ b/pkg/yqlib/doc/operators/headers/system-operators.md @@ -2,12 +2,12 @@ The `system` operator allows you to run an external command and use its output as a value in your expression. -**Security warning**: The system operator is disabled by default. You must explicitly pass `--enable-system-operator` to use it. +**Security warning**: The system operator is disabled by default. You must explicitly pass `--security-enable-system-operator` to use it. ## Usage ```bash -yq --enable-system-operator --null-input '.field = system("command"; "arg1")' +yq --security-enable-system-operator --null-input '.field = system("command"; "arg1")' ``` The operator takes: @@ -20,4 +20,4 @@ The current matched node's value is serialised and piped to the command via stdi The system operator is disabled by default. When disabled, a warning is logged and `null` is returned instead of running the command. -Use `--enable-system-operator` flag to enable it. +Use `--security-enable-system-operator` flag to enable it. diff --git a/pkg/yqlib/doc/operators/system-operators.md b/pkg/yqlib/doc/operators/system-operators.md index 765ae940f5..65501b2516 100644 --- a/pkg/yqlib/doc/operators/system-operators.md +++ b/pkg/yqlib/doc/operators/system-operators.md @@ -2,12 +2,12 @@ The `system` operator allows you to run an external command and use its output as a value in your expression. -**Security warning**: The system operator is disabled by default. You must explicitly pass `--enable-system-operator` to use it. +**Security warning**: The system operator is disabled by default. You must explicitly pass `--security-enable-system-operator` to use it. ## Usage ```bash -yq --enable-system-operator --null-input '.field = system("command"; "arg1")' +yq --security-enable-system-operator --null-input '.field = system("command"; "arg1")' ``` The operator takes: @@ -20,10 +20,10 @@ The current matched node's value is serialised and piped to the command via stdi The system operator is disabled by default. When disabled, a warning is logged and `null` is returned instead of running the command. -Use `--enable-system-operator` flag to enable it. +Use `--security-enable-system-operator` flag to enable it. ## system operator returns null when disabled -Use `--enable-system-operator` to enable the system operator. +Use `--security-enable-system-operator` to enable the system operator. Given a sample.yml file of: ```yaml @@ -39,7 +39,7 @@ country: null ``` ## Run a command with an argument -Use `--enable-system-operator` to enable the system operator. +Use `--security-enable-system-operator` to enable the system operator. Given a sample.yml file of: ```yaml @@ -47,7 +47,7 @@ country: Australia ``` then ```bash -yq --enable-system-operator '.country = system("/usr/bin/echo"; "test")' sample.yml +yq --security-enable-system-operator '.country = system("/usr/bin/echo"; "test")' sample.yml ``` will output ```yaml @@ -63,7 +63,7 @@ a: hello ``` then ```bash -yq --enable-system-operator '.a = system("/usr/bin/echo")' sample.yml +yq --security-enable-system-operator '.a = system("/usr/bin/echo")' sample.yml ``` will output ```yaml diff --git a/pkg/yqlib/operator_system.go b/pkg/yqlib/operator_system.go index 8a37d274c6..ba777f289c 100644 --- a/pkg/yqlib/operator_system.go +++ b/pkg/yqlib/operator_system.go @@ -41,7 +41,7 @@ func resolveCommandNode(commandNodes Context) (string, error) { func systemOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { if !ConfiguredSecurityPreferences.EnableSystemOps { - log.Warning("system operator is disabled, use --enable-system-operator flag to enable") + log.Warning("system operator is disabled, use --security-enable-system-operator flag to enable") results := list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) diff --git a/pkg/yqlib/operator_system_test.go b/pkg/yqlib/operator_system_test.go index 867e1921c3..4f0e4da8b2 100644 --- a/pkg/yqlib/operator_system_test.go +++ b/pkg/yqlib/operator_system_test.go @@ -17,7 +17,7 @@ func findExec(t *testing.T, name string) string { var systemOperatorDisabledScenarios = []expressionScenario{ { description: "system operator returns null when disabled", - subdescription: "Use `--enable-system-operator` to enable the system operator.", + subdescription: "Use `--security-enable-system-operator` to enable the system operator.", document: "country: Australia", expression: `.country = system("/usr/bin/echo"; "test")`, expected: []string{ @@ -54,8 +54,8 @@ func TestSystemOperatorEnabledScenarios(t *testing.T) { scenarios := []expressionScenario{ { description: "Run a command with an argument", - subdescription: "Use `--enable-system-operator` to enable the system operator.", - yqFlags: "--enable-system-operator", + subdescription: "Use `--security-enable-system-operator` to enable the system operator.", + yqFlags: "--security-enable-system-operator", document: "country: Australia", expression: `.country = system("` + echoPath + `"; "test")`, expected: []string{ @@ -65,7 +65,7 @@ func TestSystemOperatorEnabledScenarios(t *testing.T) { { description: "Run a command without arguments", subdescription: "Omit the semicolon and args to run the command with no extra arguments.", - yqFlags: "--enable-system-operator", + yqFlags: "--security-enable-system-operator", document: "a: hello", expression: `.a = system("` + echoPath + `")`, expected: []string{ From 6f94991c2ab9a1022073afc6e4978096ee89e860 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Mon, 6 Apr 2026 18:41:54 +1000 Subject: [PATCH 11/12] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- pkg/yqlib/operator_system.go | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/pkg/yqlib/operator_system.go b/pkg/yqlib/operator_system.go index ba777f289c..5f22508df5 100644 --- a/pkg/yqlib/operator_system.go +++ b/pkg/yqlib/operator_system.go @@ -9,17 +9,38 @@ import ( ) func resolveSystemArgs(argsNode *CandidateNode) []string { + if argsNode == nil { + return nil + } + if argsNode.Kind == SequenceNode { args := make([]string, 0, len(argsNode.Content)) for _, child := range argsNode.Content { + // Only non-null scalar children are valid arguments. + if child == nil { + continue + } + if child.Kind != ScalarNode || child.Tag == "!!null" { + log.Warningf("system operator: argument must be a non-null scalar; got kind=%v tag=%v - ignoring", child.Kind, child.Tag) + continue + } args = append(args, child.Value) } + if len(args) == 0 { + return nil + } return args } - if argsNode.Tag != "!!null" { - return []string{argsNode.Value} + + // Single-argument case: only accept a non-null scalar node. + if argsNode.Tag == "!!null" { + return nil + } + if argsNode.Kind != ScalarNode { + log.Warningf("system operator: args must be a non-null scalar or sequence of non-null scalars; got kind=%v tag=%v - ignoring", argsNode.Kind, argsNode.Tag) + return nil } - return nil + return []string{argsNode.Value} } func resolveCommandNode(commandNodes Context) (string, error) { From 62d28d54959751eaae752d7550de8989fbdec5e1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 8 Apr 2026 06:44:30 +0000 Subject: [PATCH 12/12] Address deep review feedback: error on disabled, strict arg/cmd validation, debug logs, docs Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/fbfba2db-60ea-4c20-a4c2-0fd396b80c81 Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> --- .../doc/operators/headers/system-operators.md | 8 +++-- pkg/yqlib/doc/operators/system-operators.md | 14 +++++--- pkg/yqlib/operator_system.go | 36 +++++++++---------- pkg/yqlib/operator_system_test.go | 17 ++++++--- 4 files changed, 45 insertions(+), 30 deletions(-) diff --git a/pkg/yqlib/doc/operators/headers/system-operators.md b/pkg/yqlib/doc/operators/headers/system-operators.md index a58fa7f30d..220a2fa395 100644 --- a/pkg/yqlib/doc/operators/headers/system-operators.md +++ b/pkg/yqlib/doc/operators/headers/system-operators.md @@ -4,6 +4,10 @@ The `system` operator allows you to run an external command and use its output a **Security warning**: The system operator is disabled by default. You must explicitly pass `--security-enable-system-operator` to use it. +**Note:** When enabled, the system operator can replicate the functionality of `env` and `load` +operators via external commands. Enabling it effectively overrides `--security-disable-env-ops` +and `--security-disable-file-ops`. + ## Usage ```bash @@ -12,12 +16,12 @@ yq --security-enable-system-operator --null-input '.field = system("command"; "a The operator takes: - A command string (required) -- An argument or array of arguments separated by `;` (optional) +- An argument (or an array of arguments), separated from the command by `;` (optional) The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string. ## Disabling the system operator -The system operator is disabled by default. When disabled, a warning is logged and `null` is returned instead of running the command. +The system operator is disabled by default. When disabled, an error is returned instead of running the command, consistent with `--security-disable-env-ops` and `--security-disable-file-ops`. Use `--security-enable-system-operator` flag to enable it. diff --git a/pkg/yqlib/doc/operators/system-operators.md b/pkg/yqlib/doc/operators/system-operators.md index 65501b2516..df1a76cd88 100644 --- a/pkg/yqlib/doc/operators/system-operators.md +++ b/pkg/yqlib/doc/operators/system-operators.md @@ -4,6 +4,10 @@ The `system` operator allows you to run an external command and use its output a **Security warning**: The system operator is disabled by default. You must explicitly pass `--security-enable-system-operator` to use it. +**Note:** When enabled, the system operator can replicate the functionality of `env` and `load` +operators via external commands. Enabling it effectively overrides `--security-disable-env-ops` +and `--security-disable-file-ops`. + ## Usage ```bash @@ -12,17 +16,17 @@ yq --security-enable-system-operator --null-input '.field = system("command"; "a The operator takes: - A command string (required) -- An argument or array of arguments separated by `;` (optional) +- An argument (or an array of arguments), separated from the command by `;` (optional) The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string. ## Disabling the system operator -The system operator is disabled by default. When disabled, a warning is logged and `null` is returned instead of running the command. +The system operator is disabled by default. When disabled, an error is returned instead of running the command, consistent with `--security-disable-env-ops` and `--security-disable-file-ops`. Use `--security-enable-system-operator` flag to enable it. -## system operator returns null when disabled +## system operator returns error when disabled Use `--security-enable-system-operator` to enable the system operator. Given a sample.yml file of: @@ -34,8 +38,8 @@ then yq '.country = system("/usr/bin/echo"; "test")' sample.yml ``` will output -```yaml -country: null +```bash +Error: system operations are disabled, use --security-enable-system-operator to enable ``` ## Run a command with an argument diff --git a/pkg/yqlib/operator_system.go b/pkg/yqlib/operator_system.go index 5f22508df5..c9937957e0 100644 --- a/pkg/yqlib/operator_system.go +++ b/pkg/yqlib/operator_system.go @@ -8,9 +8,9 @@ import ( "strings" ) -func resolveSystemArgs(argsNode *CandidateNode) []string { +func resolveSystemArgs(argsNode *CandidateNode) ([]string, error) { if argsNode == nil { - return nil + return nil, nil } if argsNode.Kind == SequenceNode { @@ -21,26 +21,24 @@ func resolveSystemArgs(argsNode *CandidateNode) []string { continue } if child.Kind != ScalarNode || child.Tag == "!!null" { - log.Warningf("system operator: argument must be a non-null scalar; got kind=%v tag=%v - ignoring", child.Kind, child.Tag) - continue + return nil, fmt.Errorf("system operator: argument must be a non-null scalar; got kind=%v tag=%v", child.Kind, child.Tag) } args = append(args, child.Value) } if len(args) == 0 { - return nil + return nil, nil } - return args + return args, nil } // Single-argument case: only accept a non-null scalar node. if argsNode.Tag == "!!null" { - return nil + return nil, nil } if argsNode.Kind != ScalarNode { - log.Warningf("system operator: args must be a non-null scalar or sequence of non-null scalars; got kind=%v tag=%v - ignoring", argsNode.Kind, argsNode.Tag) - return nil + return nil, fmt.Errorf("system operator: args must be a non-null scalar or sequence of non-null scalars; got kind=%v tag=%v", argsNode.Kind, argsNode.Tag) } - return []string{argsNode.Value} + return []string{argsNode.Value}, nil } func resolveCommandNode(commandNodes Context) (string, error) { @@ -51,7 +49,7 @@ func resolveCommandNode(commandNodes Context) (string, error) { log.Debugf("system operator: command expression returned %d results, using first", commandNodes.MatchingNodes.Len()) } cmdNode := commandNodes.MatchingNodes.Front().Value.(*CandidateNode) - if cmdNode.Kind != ScalarNode || cmdNode.Tag == "!!null" { + if cmdNode.Kind != ScalarNode || cmdNode.guessTagFromCustomType() != "!!str" { return "", fmt.Errorf("system operator: command must be a string scalar") } if cmdNode.Value == "" { @@ -62,13 +60,7 @@ func resolveCommandNode(commandNodes Context) (string, error) { func systemOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { if !ConfiguredSecurityPreferences.EnableSystemOps { - log.Warning("system operator is disabled, use --security-enable-system-operator flag to enable") - results := list.New() - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - results.PushBack(candidate.CreateReplacement(ScalarNode, "!!null", "null")) - } - return context.ChildContext(results), nil + return Context{}, fmt.Errorf("system operations are disabled, use --security-enable-system-operator to enable") } // determine at parse time whether we have (command; args) or just (command) @@ -98,8 +90,14 @@ func systemOperator(d *dataTreeNavigator, context Context, expressionNode *Expre if err != nil { return Context{}, err } + if argsNodes.MatchingNodes.Len() > 1 { + log.Debugf("system operator: args expression returned %d results, using first", argsNodes.MatchingNodes.Len()) + } if argsNodes.MatchingNodes.Front() != nil { - args = resolveSystemArgs(argsNodes.MatchingNodes.Front().Value.(*CandidateNode)) + args, err = resolveSystemArgs(argsNodes.MatchingNodes.Front().Value.(*CandidateNode)) + if err != nil { + return Context{}, err + } } } else { commandNodes, err := d.GetMatchingNodes(nodeContext, expressionNode.RHS) diff --git a/pkg/yqlib/operator_system_test.go b/pkg/yqlib/operator_system_test.go index 4f0e4da8b2..46a0ecd4b5 100644 --- a/pkg/yqlib/operator_system_test.go +++ b/pkg/yqlib/operator_system_test.go @@ -16,13 +16,11 @@ func findExec(t *testing.T, name string) string { var systemOperatorDisabledScenarios = []expressionScenario{ { - description: "system operator returns null when disabled", + description: "system operator returns error when disabled", subdescription: "Use `--security-enable-system-operator` to enable the system operator.", document: "country: Australia", expression: `.country = system("/usr/bin/echo"; "test")`, - expected: []string{ - "D0, P[], (!!map)::country: null\n", - }, + expectedError: "system operations are disabled, use --security-enable-system-operator to enable", }, } @@ -105,6 +103,17 @@ func TestSystemOperatorEnabledScenarios(t *testing.T) { expression: `.a = system(null)`, expectedError: "system operator: command must be a string scalar", }, + { + description: "System operator processes multiple matched nodes", + skipDoc: true, + document: "a: first", + document2: "a: second", + expression: `.a = system("` + echoPath + `"; "replaced")`, + expected: []string{ + "D0, P[], (!!map)::a: replaced\n", + "D0, P[], (!!map)::a: replaced\n", + }, + }, } for _, tt := range scenarios {