WPB-24072 mount galley.yaml into background-worker

changelog.d/0-release-notes/WPB-24072

@@ -0,0 +1 @@
+`background-worker` now reuses `galley`'s configmap and secrets for cassandra, postgres and federation domain settings. This removes redundant settings and keeps the two services aligned. No operator action is strictly required, however we advice to remove the value overrides of the `background-worker` for galley's cassandra, postgres, and the federation domain as they are duplicate an no longer needed.

charts/wire-server/templates/_helpers.tpl

@@ -39,14 +39,6 @@
 {{- end -}}
 {{- end -}}
 
-{{- define "galleyTlsSecretRef" -}}
-{{- if and .cassandraGalley .cassandraGalley.tlsCaSecretRef -}}
-{{ .cassandraGalley.tlsCaSecretRef | toYaml }}
-{{- else }}
-{{- dict "name" "background-worker-cassandra-galley" "key" "ca.pem" | toYaml -}}
-{{- end -}}
-{{- end -}}
-
 {{/* BRIG */}}
 {{- define "brig.tlsSecretRef" -}}
 {{- if .cassandra.tlsCaSecretRef -}}

charts/wire-server/templates/background-worker/cassandra-secret.yaml

@@ -29,18 +29,3 @@ type: Opaque
 data:
   ca.pem: {{ $backgroundWorker.config.cassandraBrig.tlsCa | b64enc | quote }}
 {{- end }}
-{{- if and $backgroundWorker.config.cassandraGalley (not (empty $backgroundWorker.config.cassandraGalley.tlsCa)) }}
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: background-worker-cassandra-galley
-  labels:
-    app: background-worker
-    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
-    release: "{{ .Release.Name }}"
-    heritage: "{{ .Release.Service }}"
-type: Opaque
-data:
-  ca.pem: {{ $backgroundWorker.config.cassandraGalley.tlsCa | b64enc | quote }}
-{{- end }}

charts/wire-server/templates/background-worker/configmap.yaml

@@ -62,28 +62,6 @@ data:
       {{- end }}
     {{- end }}
 
-    {{- if .cassandraGalley }}
-    cassandraGalley:
-      endpoint:
-        host: {{ .cassandraGalley.host }}
-        port: 9042
-      keyspace: galley
-      {{- if hasKey .cassandraGalley "filterNodesByDatacentre" }}
-      filterNodesByDatacentre: {{ .cassandraGalley.filterNodesByDatacentre }}
-      {{- end }}
-      {{- if eq (include "useCassandraTLS" .cassandraGalley) "true" }}
-      tlsCa: /etc/wire/background-worker/cassandra-galley/{{- (include "galleyTlsSecretRef" . | fromYaml).key }}
-      {{- end }}
-    {{- end }}
-
-    postgresql: {{ toYaml .postgresql | nindent 6 }}
-    postgresqlPool: {{ toYaml .postgresqlPool | nindent 6 }}
-    {{- if hasKey $backgroundWorker.secrets "pgPassword" }}
-    postgresqlPassword: /etc/wire/background-worker/secrets/pgPassword
-    {{- end }}
-
-    federationDomain: {{ $backgroundWorker.config.federationDomain }}
-
     {{- with .rabbitmq }}
     rabbitmq:
       host: {{ .host }}
@@ -95,8 +73,8 @@ data:
       {{- end }}
       enableTls: {{ .enableTls }}
       insecureSkipVerifyTls: {{ .insecureSkipVerifyTls }}
-      {{- if .tlsCaSecretRef }}
-      caCert: /etc/wire/background-worker/rabbitmq-ca/{{ .tlsCaSecretRef.key }}
+      {{- if $.Values.galley.config.rabbitmq.tlsCaSecretRef }}
+      caCert: /etc/wire/galley/rabbitmq-ca/{{ $.Values.galley.config.rabbitmq.tlsCaSecretRef.key }}
       {{- end }}
     {{- end }}
 

charts/wire-server/templates/background-worker/deployment.yaml

@@ -26,8 +26,10 @@ spec:
       annotations:
         # An annotation of the configmap checksum ensures changes to the configmap cause a redeployment upon `helm upgrade`
         checksum/configmap: {{ include (print .Template.BasePath "/background-worker/configmap.yaml") . | sha256sum }}
+        checksum/galley-configmap: {{ include (print .Template.BasePath "/galley/configmap.yaml") . | sha256sum }}
         checksum/secret: {{ include (print .Template.BasePath "/background-worker/secret.yaml") . | sha256sum }}
         checksum/cassandra-secret: {{ include (print .Template.BasePath "/background-worker/cassandra-secret.yaml") . | sha256sum }}
+        checksum/galley-secret: {{ include (print .Template.BasePath "/galley/secret.yaml") . | sha256sum }}
         fluentbit.io/parser: json
     spec:
       serviceAccount: null
@@ -37,9 +39,15 @@ spec:
         - name: "background-worker-config"
           configMap:
             name: "background-worker"
+        - name: "galley-config"
+          configMap:
+            name: "galley"
         - name: "background-worker-secrets"
           secret:
             secretName: "background-worker"
+        - name: "galley-secrets"
+          secret:
+            secretName: "galley"
         {{- if eq (include "useCassandraTLS" $backgroundWorker.config.cassandra) "true" }}
         - name: "background-worker-cassandra-gundeck"
           secret:
@@ -50,15 +58,15 @@ spec:
           secret:
             secretName: {{ (include "brigTlsSecretRef" $backgroundWorker.config | fromYaml).name }}
         {{- end }}
-        {{- if eq (include "useCassandraTLS" $backgroundWorker.config.cassandraGalley) "true" }}
-        - name: "background-worker-cassandra-galley"
+        {{- if eq (include "useCassandraTLS" .Values.galley.config.cassandra) "true" }}
+        - name: "galley-cassandra"
           secret:
-            secretName: {{ (include "galleyTlsSecretRef" $backgroundWorker.config | fromYaml).name }}
+            secretName: {{ (include "galley.tlsSecretRef" .Values.galley.config | fromYaml).name }}
         {{- end }}
-        {{- if $backgroundWorker.config.rabbitmq.tlsCaSecretRef }}
-        - name: "rabbitmq-ca"
+        {{- if .Values.galley.config.rabbitmq.tlsCaSecretRef }}
+        - name: "galley-rabbitmq-ca"
           secret:
-            secretName: {{ $backgroundWorker.config.rabbitmq.tlsCaSecretRef.name }}
+            secretName: {{ .Values.galley.config.rabbitmq.tlsCaSecretRef.name }}
         {{- end }}
         {{- if .Values.additionalVolumes }}
         {{ toYaml .Values.additionalVolumes | nindent 8 }}
@@ -74,8 +82,12 @@ spec:
           volumeMounts:
           - name: "background-worker-secrets"
             mountPath: "/etc/wire/background-worker/secrets"
+          - name: "galley-secrets"
+            mountPath: "/etc/wire/galley/secrets"
           - name: "background-worker-config"
             mountPath: "/etc/wire/background-worker/conf"
+          - name: "galley-config"
+            mountPath: "/etc/wire/galley/conf"
           {{- if eq (include "useCassandraTLS" $backgroundWorker.config.cassandra) "true" }}
           - name: "background-worker-cassandra-gundeck"
             mountPath: "/etc/wire/background-worker/cassandra-gundeck"
@@ -84,13 +96,13 @@ spec:
           - name: "background-worker-cassandra-brig"
             mountPath: "/etc/wire/background-worker/cassandra-brig"
           {{- end }}
-          {{- if eq (include "useCassandraTLS" $backgroundWorker.config.cassandraGalley) "true" }}
-          - name: "background-worker-cassandra-galley"
-            mountPath: "/etc/wire/background-worker/cassandra-galley"
+          {{- if eq (include "useCassandraTLS" .Values.galley.config.cassandra) "true" }}
+          - name: "galley-cassandra"
+            mountPath: "/etc/wire/galley/cassandra"
           {{- end }}
-          {{- if $backgroundWorker.config.rabbitmq.tlsCaSecretRef }}
-          - name: "rabbitmq-ca"
-            mountPath: "/etc/wire/background-worker/rabbitmq-ca/"
+          {{- if .Values.galley.config.rabbitmq.tlsCaSecretRef }}
+          - name: "galley-rabbitmq-ca"
+            mountPath: "/etc/wire/galley/rabbitmq-ca/"
           {{- end }}
           {{- if .Values.additionalVolumeMounts }}
           {{ toYaml .Values.additionalVolumeMounts | nindent 10 }}

charts/wire-server/values.yaml

@@ -938,34 +938,9 @@ background-worker:
     # Cassandra clusters used by background-worker
     cassandra:
       host: aws-cassandra
-    cassandraGalley:
-      host: aws-cassandra
     cassandraBrig:
       host: aws-cassandra
 
-    # Postgres connection settings
-    #
-    # Values are described in https://www.postgresql.org/docs/17/libpq-connect.html#LIBPQ-PARAMKEYWORDS
-    # To set the password via a brig secret see `secrets.pgPassword`.
-    #
-    # `additionalVolumeMounts` and `additionalVolumes` can be used to mount
-    # additional files (e.g. certificates) into the brig container. This way
-    # does not work for password files (parameter `passfile`), because
-    # libpq-connect requires access rights (mask 0600) for them that we cannot
-    # provide for random uids.
-    #
-    # Below is an example configuration we're using for our CI tests.
-    postgresql:
-      host: postgresql # DNS name without protocol
-      port: "5432"
-      user: wire-server
-      dbname: wire-server
-    postgresqlPool:
-      size: 5
-      acquisitionTimeout: 10s
-      agingTimeout: 1d
-      idlenessTimeout: 10m
-
     # Setting this to `true` will start conversation migration to postgresql.
     #
     # NOTE: It is very important that galley be configured to with
@@ -1006,7 +981,6 @@ background-worker:
 
   secrets:
     {}
-    # pgPassword: <postgres-password>
 
   podSecurityContext:
     allowPrivilegeEscalation: false

hack/helm_vars/wire-server/values.yaml.gotmpl

@@ -635,11 +635,6 @@ background-worker:
       pushBackoffMinWait: 1000 # 1ms
       pushBackoffMaxWait: 500000 # 0.5s
       remotesRefreshInterval: 1000000 # 1s
-    postgresql:
-      host: "postgresql"
-      port: "5432"
-      user: wire-server
-      dbname: wire-server
     backgroundJobs:
       concurrency: 8
       jobTimeout: 60s
@@ -653,14 +648,6 @@ background-worker:
           name: "cassandra-jks-keystore"
           key: "ca.crt"
       {{- end }}
-    cassandraGalley:
-      host: {{ .Values.cassandraHost }}
-      replicaCount: 1
-      {{- if .Values.useK8ssandraSSL.enabled }}
-      tlsCaSecretRef:
-          name: "cassandra-jks-keystore"
-          key: "ca.crt"
-      {{- end }}
     cassandraBrig:
       host: {{ .Values.cassandraHost }}
       replicaCount: 1
@@ -669,8 +656,6 @@ background-worker:
           name: "cassandra-jks-keystore"
           key: "ca.crt"
       {{- end }}
-    # See helmfile for the real value
-    federationDomain: integration.example.com
     postgresMigration:
       conversation: {{ .Values.conversationStore }}
       conversationCodes: {{ .Values.conversationCodesStore }}
@@ -684,7 +669,6 @@ background-worker:
         name: "rabbitmq-certificate"
         key: "ca.crt"
   secrets:
-    pgPassword: "posty-the-gres"
     rabbitmq:
       username: {{ .Values.rabbitmqUsername }}
       password: {{ .Values.rabbitmqPassword }}

hack/helmfile.yaml.gotmpl

@@ -324,8 +324,6 @@ releases:
         value: {{ .Values.federationDomain1 }}
       - name: galley.config.settings.federationDomain
         value: {{ .Values.federationDomain1 }}
-      - name: backgroundWorker.config.federationDomain
-        value: {{ .Values.federationDomain1 }}
       - name: cargohold.config.settings.federationDomain
         value: {{ .Values.federationDomain1 }}
       - name: brig.config.wireServerEnterprise.enabled
@@ -346,8 +344,6 @@ releases:
         value: {{ .Values.federationDomain2 }}
       - name: galley.config.settings.federationDomain
         value: {{ .Values.federationDomain2 }}
-      - name: backgroundWorker.config.federationDomain
-        value: {{ .Values.federationDomain2 }}
       - name: cargohold.config.settings.federationDomain
         value: {{ .Values.federationDomain2 }}
     needs:

integration/test/Testlib/ModService.hs

@@ -439,23 +439,27 @@ processColors =
 
 data ServiceInstance = ServiceInstance
   { handle :: ProcessHandle,
-    config :: FilePath
+    configs :: [FilePath]
   }
 
 timeout :: Int -> IO a -> IO (Maybe a)
 timeout usecs action = either (const Nothing) Just <$> race (threadDelay usecs) action
 
 cleanupService :: (HasCallStack) => ServiceInstance -> IO ()
-cleanupService inst = do
-  mPid <- getPid inst.handle
-  for_ mPid (signalProcess keyboardSignal)
-  timeout 50000 (waitForProcess inst.handle) >>= \case
-    Just _ -> pure ()
-    Nothing -> do
-      for_ mPid (signalProcess killProcess)
-      void $ waitForProcess inst.handle
-  whenM (doesFileExist inst.config) $ removeFile inst.config
-  whenM (doesDirectoryExist inst.config) $ removeDirectoryRecursive inst.config
+cleanupService inst = stopProcesses `E.finally` cleanupPaths
+  where
+    stopProcesses = do
+      mPid <- getPid inst.handle
+      for_ mPid (signalProcess keyboardSignal)
+      timeout 50000 (waitForProcess inst.handle) >>= \case
+        Just _ -> pure ()
+        Nothing -> do
+          for_ mPid (signalProcess killProcess)
+          void $ waitForProcess inst.handle
+    cleanupPaths =
+      for_ inst.configs $ \path -> do
+        whenM (doesFileExist path) $ removeFile path
+        whenM (doesDirectoryExist path) $ removeDirectoryRecursive path
 
 -- | Wait for a service to come up.
 waitUntilServiceIsUp :: (HasCallStack) => Maybe ProcessDebug -> String -> Service -> App ()
@@ -501,10 +505,23 @@ withProcess resource overrides service = do
   phRef <- liftIO $ newIORef Nothing
 
   getConfig <- lift $ readAndUpdateConfig overrides resource service
+  getGalleyConf <- lift $ readAndUpdateConfig overrides resource Galley
   let prefix = "[" <> execName <> "@" <> domain <> maybe "" (":" <>) env.currentTestName <> "] "
   let initProcess = case (service, cwd) of
         (Nginz, Nothing) -> startNginzK8s domain sm
         (Nginz, Just _) -> startNginzLocalIO
+        (BackgroundWorker, _) -> do
+          config <- getConfig
+          galleyConf <- getGalleyConf
+          tempFile <- writeTempFile "/tmp" (execName <> "-" <> domain <> "-" <> ".yaml") (cs $ Yaml.encode config)
+          galleyConfTemp <- writeTempFile "/tmp" (configName Galley <> "-" <> domain <> "-" <> ".yaml") (cs $ Yaml.encode galleyConf)
+          let params = ["-c", tempFile, "--galley-config-file", galleyConfTemp]
+          (_, Just stdoutHdl, Just stderrHdl, ph) <- createProcess (proc exe params) {cwd = cwd, std_out = CreatePipe, std_err = CreatePipe}
+          let colorize = fromMaybe id (lookup execName processColors)
+          void $ forkIO $ logToConsoleDebug (Just stdOut) colorize prefix stdoutHdl
+          void $ forkIO $ logToConsoleDebug (Just stdErr) colorize prefix stderrHdl
+          liftIO $ writeIORef phRef (Just ph)
+          pure $ ServiceInstance ph [tempFile, galleyConfTemp]
         _ -> do
           config <- getConfig
           tempFile <- writeTempFile "/tmp" (execName <> "-" <> domain <> "-" <> ".yaml") (cs $ Yaml.encode config)
@@ -513,7 +530,7 @@ withProcess resource overrides service = do
           void $ forkIO $ logToConsoleDebug (Just stdOut) colorize prefix stdoutHdl
           void $ forkIO $ logToConsoleDebug (Just stdErr) colorize prefix stderrHdl
           liftIO $ writeIORef phRef (Just ph)
-          pure $ ServiceInstance ph tempFile
+          pure $ ServiceInstance ph [tempFile]
 
   void $
     hoistCodensity $
@@ -600,7 +617,7 @@ startNginzK8s domain sm = do
   Text.writeFile nginxConfFile $ replaceUpstreamsInConfig conf' sm
 
   ph <- startNginz domain nginxConfFile "/"
-  pure $ ServiceInstance ph tmpDir
+  pure $ ServiceInstance ph [tmpDir]
 
 startNginzLocal :: BackendResource -> App ServiceInstance
 startNginzLocal resource = do
@@ -665,7 +682,7 @@ startNginzLocal resource = do
   ph <- liftIO $ startNginz domain nginxConfFile tmpDir
 
   -- return handle and nginx tmp dir path
-  pure $ ServiceInstance ph tmpDir
+  pure $ ServiceInstance ph [tmpDir]
 
 makeUpstreamsCfgs :: ServiceMap -> String
 makeUpstreamsCfgs sm =

libs/types-common/src/Util/Options.hs

@@ -118,20 +118,9 @@ getOptions desc mp defaultPath = do
         (header desc <> fullDesc)
   exists <- doesFileExist path
   case (exists, mOpts) of
-    -- config file exists, take options from there
-    (True, _) -> do
-      decodeFileEither path >>= \case
-        Left e ->
-          fail $
-            show e
-              <> " while attempting to decode "
-              <> path
-        Right o -> pure o
-    -- config doesn't exist, take options from command line
+    (True, _) -> decodeConfigFile path
     (False, Just opts) -> pure opts
-    -- no config, no parser, just fail
-    (False, Nothing) ->
-      fail $ "Config file at " <> path <> " does not exist."
+    (False, Nothing) -> fail $ "Config file at " <> path <> " does not exist."
   where
     optsOrConfigFile :: Parser (FilePath, Maybe a)
     optsOrConfigFile =
@@ -145,6 +134,21 @@ getOptions desc mp defaultPath = do
           )
         <*> sequenceA mp
 
+decodeConfigFile :: (FromJSON a) => FilePath -> IO a
+decodeConfigFile path = do
+  exists <- doesFileExist path
+  if exists
+    then
+      decodeFileEither path >>= \case
+        Left e ->
+          fail $
+            show e
+              <> " while attempting to decode "
+              <> path
+        Right o -> pure o
+    else
+      fail $ "Config file at " <> path <> " does not exist."
+
 parseAWSEndpoint :: ReadM AWSEndpoint
 parseAWSEndpoint = readerAsk >>= maybe (error "Could not parse AWS endpoint") pure . fromByteString . fromString