Skip to content

Fix SHC appframework bundle-push loop in FIPS mode #1672

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
dpericaxon wants to merge 2 commits into from
Closed

Fix SHC appframework bundle-push loop in FIPS mode #1672

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
83288f4
Fix SHC appframework bundle-push loop in FIPS mode
dpericaxon Jan 21, 2026
bd9a89a
Normalize SHC bundle-push status output using a benign banner list
dpericaxon Jan 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 79 additions & 12 deletions pkg/splunk/enterprise/afwscheduler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1676,15 +1676,60 @@ func (shcPlaybookContext *SHCPlaybookContext) isBundlePushComplete(ctx context.C
return false, err
}

// NOTE: In some environments (notably Splunk in FIPS mode), the Splunk CLI emits a banner
// like:
// "FIPS provider enabled. ..."
// and sometimes a CLI TLS warning, before the actual bundle push status line is written.
//
// The operator runs `apply shcluster-bundle ... &> status_file &` asynchronously and then
// polls the status file. If we treat "banner-only" output as an error, we can prematurely
// reset state to Pending and re-trigger bundle push, causing collisions like:
// ConfDeploymentException: Can't start deployment job as one is already running!
//
// So we normalize banner-only / known benign warning lines away and only treat remaining
// output as an error if it isn't the known success string.
normalizedOut := normalizeSHCBundlePushStatusOutput(stdOut)

// Check if we did not get the desired output in the status file. There can be 2 scenarios -
// 1. stdOut is empty, which means bundle push is still in progress
Copy link
Copy Markdown
Collaborator

@kasiakoziol kasiakoziol Jan 23, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please update the comment for consistency - stdOut to normalizedOut

Copy link
Copy Markdown
Author

@dpericaxon dpericaxon Jan 23, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done! Updated the comment to reference normalizedOut for consistency with the normalized parsing logic.

// 2. stdOut has some other string other than the bundle push success message
if stdOut == "" {
if normalizedOut == "" {
scopedLog.Info("SHC Bundle Push is still in progress")
return false, nil
} else if !strings.Contains(stdOut, shcBundlePushCompleteStr) {
}

// The completion constant includes a trailing newline, but we strip whitespace in normalizeSHCBundlePushStatusOutput,
// so check for the trimmed message.
completeStr := strings.TrimSpace(shcBundlePushCompleteStr)

// Happy path: bundle push complete.
if strings.Contains(normalizedOut, completeStr) {
// now that bundle push is complete, remove the status file
err = shcPlaybookContext.removeSHCBundlePushStatusFile(ctx)
if err != nil {
scopedLog.Error(err, "removing SHC Bundle Push status file failed, will retry again.")

// reset the state to BundlePushInProgress so that we can check the status of file again.
setBundlePushState(ctx, shcPlaybookContext.afwPipeline, enterpriseApi.BundlePushInProgress)

// don't return error from here, so that we can retry cleaning the file in next run
return false, nil
}

return true, nil
}

// If Splunk reports that a deployment job is already running, treat this as "still in progress"
// (don't reset to Pending / retrigger bundle push, which can create a retry storm).
if strings.Contains(normalizedOut, "ConfDeploymentException: Can't start deployment job as one is already running!") {
scopedLog.Info("SHC Bundle Push appears to already be running; will recheck status", "statusFileOutput", normalizedOut)
return false, nil
}

// Any other non-empty output that isn't the known success string is treated as an error.
if !strings.Contains(normalizedOut, completeStr) {
// this means there was an error in bundle push command
err = fmt.Errorf("there was an error in applying SHC Bundle, err=\"%v\"", stdOut)
err = fmt.Errorf("there was an error in applying SHC Bundle, err=\"%v\"", normalizedOut)
scopedLog.Error(err, "SHC Bundle push status file reported an error while applying bundle")

// reset the bundle push state to Pending, so that we retry again.
Expand All @@ -1698,19 +1743,41 @@ func (shcPlaybookContext *SHCPlaybookContext) isBundlePushComplete(ctx context.C
return false, err
}

// now that bundle push is complete, remove the status file
err = shcPlaybookContext.removeSHCBundlePushStatusFile(ctx)
if err != nil {
scopedLog.Error(err, "removing SHC Bundle Push status file failed, will retry again.")
// Defensive fallback: treat as still in progress.
return false, nil
}

// reset the state to BundlePushInProgress so that we can check the status of file again.
setBundlePushState(ctx, shcPlaybookContext.afwPipeline, enterpriseApi.BundlePushInProgress)
// normalizeSHCBundlePushStatusOutput strips known benign Splunk CLI banner/warning lines
// from the SHC bundle push status file output. This allows the operator to treat
// "banner-only" output as still-in-progress rather than as an error.
func normalizeSHCBundlePushStatusOutput(stdOut string) string {
out := strings.TrimSpace(stdOut)
if out == "" {
return ""
}

// don't return error from here, so that we can retry cleaning the file in next run
return false, nil
lines := strings.Split(out, "\n")
kept := make([]string, 0, len(lines))
for _, raw := range lines {
line := strings.TrimSpace(raw)
if line == "" {
continue
}

// FIPS banner emitted by Splunk CLI in FIPS-enabled environments
Copy link
Copy Markdown
Collaborator

@rlieberman-splunk rlieberman-splunk Jan 22, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a list of benign banners somewhere that we can add to the checks here? I'm worried if we merge this it will only cover a specific scenario with hard coded values.

Copy link
Copy Markdown
Collaborator

@kasiakoziol kasiakoziol Jan 23, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we don't find out, then as of now, we can just make a list of such banners (as of now 1-element), use this list in the code to validate the output and once we learn more, we can expand the list with new banners.

Copy link
Copy Markdown
Author

@dpericaxon dpericaxon Jan 23, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@rlieberman-splunk @kasiakoziol is this an action item on your side to dig into this? For now I Implemented a benignLinePrefixes list in normalizeSHCBundlePushStatusOutput() (currently includes the known FIPS banner and the TLS hostname validation warning). This keeps the logic extensible and we can expand the list as additional benign lines are identified.Implemented a benignLinePrefixes list in normalizeSHCBundlePushStatusOutput() (currently includes the known FIPS banner and the TLS hostname validation warning). This keeps the logic extensible and we can expand the list as additional benign lines are identified.

if strings.HasPrefix(line, "FIPS provider enabled.") {
continue
}

// Common benign CLI warning (often appears right after the FIPS banner)
if strings.HasPrefix(line, "WARNING: Server Certificate Hostname Validation is disabled.") {
continue
}

kept = append(kept, line)
}

return true, nil
return strings.TrimSpace(strings.Join(kept, "\n"))
}

// triggerBundlePush triggers the bundle push operation for SHC
Expand Down
18 changes: 18 additions & 0 deletions pkg/splunk/enterprise/afwscheduler_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2875,6 +2875,24 @@ func TestSHCRunPlaybook(t *testing.T) {
t.Errorf("runPlaybook() should not have returned error or wrong bundle push state, err=%v, bundle push state=%s", err, bundlePushStateAsStr(ctx, getBundlePushState(afwPipeline)))
}

// Test7b: Bundle push is still in progress when the status file has only benign Splunk CLI banner/warning
// (e.g. FIPS banner), and does not yet include the bundle push completion string.
appDeployContext.BundlePushStatus.BundlePushStage = enterpriseApi.BundlePushInProgress
mockPodExecReturnContexts[2].StdOut = "FIPS provider enabled. name: OpenSSL FIPS Provider, version: 3.0.9, buildinfo: 3.0.9, status: Success\nWARNING: Server Certificate Hostname Validation is disabled. Please see server.conf/[sslConfig]/cliVerifyServerName for details.\n"
err = playbookContext.runPlaybook(ctx)
if err != nil || getBundlePushState(afwPipeline) != enterpriseApi.BundlePushInProgress {
t.Errorf("runPlaybook() should not have returned error for banner-only output; err=%v, bundle push state=%s", err, bundlePushStateAsStr(ctx, getBundlePushState(afwPipeline)))
}

// Test7c: Bundle push is still in progress when Splunk reports a deployment job is already running.
// This should NOT reset the state to Pending (which would retrigger and create a retry storm).
appDeployContext.BundlePushStatus.BundlePushStage = enterpriseApi.BundlePushInProgress
mockPodExecReturnContexts[2].StdOut = "FIPS provider enabled. name: OpenSSL FIPS Provider, version: 3.0.9, buildinfo: 3.0.9, status: Success\nConfDeploymentException: Can't start deployment job as one is already running!\n"
err = playbookContext.runPlaybook(ctx)
if err != nil || getBundlePushState(afwPipeline) != enterpriseApi.BundlePushInProgress {
t.Errorf("runPlaybook() should not have returned error for ConfDeploymentException already-running output; err=%v, bundle push state=%s", err, bundlePushStateAsStr(ctx, getBundlePushState(afwPipeline)))
}

// Test8: Bundle push is still in progress since stdOut != shcBundlePushCompleteStr
mockPodExecReturnContexts[2].StdOut = "Error while deploying apps"
err = playbookContext.runPlaybook(ctx)
Expand Down
Loading