CSPL-4623: FIPS appframework fix for SHC and ES

.github/workflows/int-test-workflow.yml

@@ -194,7 +194,7 @@ jobs:
           mkdir -p ./bin
           cp /snap/bin/kustomize ./bin/kustomize
       - name: Run Integration test
-        timeout-minutes: 240
+        timeout-minutes: 540
         env:
           TEST_TIMEOUT: 225m
           TEST_S3_ACCESS_KEY_ID: ${{ vars.TEST_S3_ACCESS_KEY_ID }}

pkg/splunk/enterprise/afwscheduler.go

@@ -1688,13 +1688,45 @@ func (shcPlaybookContext *SHCPlaybookContext) isBundlePushComplete(ctx context.C
 		return false, err
 	}
 
-	// Check if we did not get the desired output in the status file. There can be 2 scenarios -
+	// Check if we did not get the desired output in the status file. There can be 3 scenarios -
 	// 1. stdOut is empty, which means bundle push is still in progress
-	// 2. stdOut has some other string other than the bundle push success message
+	// 2. stdOut contains only informational lines (e.g. the FIPS provider banner written to
+	//    stderr by the Splunk CLI on FIPS-enabled clusters, captured via the &> shell redirect
+	//    in applySHCBundleCmdStr before the actual push output is written)
+	// 3. stdOut has some other string other than the bundle push success message
 	if stdOut == "" {
 		scopedLog.Info("SHC Bundle Push is still in progress")
 		return false, nil
 	} else if !strings.Contains(stdOut, shcBundlePushCompleteStr) {
+		// Check whether the file contains only known informational lines. On FIPS-enabled
+		// clusters the Splunk binary immediately writes the FIPS provider banner (and SSL
+		// warnings) to stderr at startup; because the bundle push command uses &> to
+		// redirect all output to the status file, these lines appear in the file before the
+		// actual push result. Treat such content as "still in progress" so we do not
+		// prematurely abort a running push and trigger a retry storm.
+		//
+		// IMPORTANT: SSL certificate warnings ("WARNING: Server Certificate ...") are only
+		// treated as informational when the FIPS provider banner is also present. On non-FIPS
+		// clusters the Splunk CLI can also emit SSL warnings (e.g. when hostname validation is
+		// disabled), but if those warnings are the only content in the status file it means the
+		// push failed silently — we must fall through to error/retry rather than waiting forever.
+		hasFIPSContent := strings.Contains(stdOut, splunkFIPSProviderBannerStr)
+		hasMeaningfulContent := false
+		for _, line := range strings.Split(stdOut, "\n") {
+			trimmed := strings.TrimSpace(line)
+			if trimmed == "" ||
+				strings.HasPrefix(trimmed, splunkFIPSProviderBannerStr) ||
+				(hasFIPSContent && strings.HasPrefix(trimmed, splunkSSLCertWarnStr)) {
+				continue
+			}
+			hasMeaningfulContent = true
+			break
+		}
+		if !hasMeaningfulContent {
+			scopedLog.Info("SHC Bundle Push is still in progress (status file contains only informational messages)")
+			return false, nil
+		}
+
 		// this means there was an error in bundle push command
 		err = fmt.Errorf("there was an error in applying SHC Bundle, err=\"%v\"", stdOut)
 		scopedLog.Error(err, "SHC Bundle push status file reported an error while applying bundle")
@@ -2078,7 +2110,16 @@ func handleEsappPostinstall(rctx context.Context, preCtx *premiumAppScopePlayboo
 
 	streamOptions := splutil.NewStreamOptionsObject(command)
 	stdOut, stdErr, err := preCtx.localCtx.podExecClient.RunPodExecCommand(rctx, streamOptions, []string{"/bin/sh"})
-	if stdErr != "" || err != nil {
+
+	// Log stderr content for debugging but don't use it for error detection.
+	// On FIPS-enabled clusters the Splunk CLI always writes the FIPS provider
+	// banner and related informational messages to stderr on every invocation,
+	// so a non-empty stderr does not indicate failure.
+	if stdErr != "" {
+		scopedLog.Info("Post install command stderr output (informational only)", "stdout", stdOut, "stderr", stdErr, "post install command", command)
+	}
+
+	if err != nil {
 		phaseInfo.FailCount++
 		scopedLog.Error(err, "premium scoped app package install failed", "stdout", stdOut, "stderr", stdErr, "post install command", command, "failCount", phaseInfo.FailCount)
 		return fmt.Errorf("premium scoped app package install failed. stdOut: %s, stdErr: %s, post install command: %s, failCount: %d", stdOut, stdErr, command, phaseInfo.FailCount)