diff --git a/auth/access_token.go b/auth/access_token.go index 7e3517996..77705fd37 100644 --- a/auth/access_token.go +++ b/auth/access_token.go @@ -22,9 +22,6 @@ import ( "strings" authnv1 "k8s.io/api/authentication/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - "sigs.k8s.io/controller-runtime/pkg/client" "github.com/fluxcd/pkg/cache" ) @@ -45,49 +42,57 @@ func GetAccessToken(ctx context.Context, provider Provider, opts ...Option) (Tok } // Update access token fetcher for a service account if specified. - var serviceAccount *corev1.ServiceAccount - var providerIdentity string - var audiences []string - if o.ShouldGetServiceAccountToken() { + var saInfo *serviceAccountInfo + if o.ShouldGetServiceAccount() { + // Check the feature gate for object-level workload identity. + if !IsObjectLevelWorkloadIdentityEnabled() { + return nil, ErrObjectLevelWorkloadIdentityNotEnabled + } + // Fetch service account details. var err error - saRef := client.ObjectKey{ - Name: o.ServiceAccountName, - Namespace: o.ServiceAccountNamespace, - } - serviceAccount, audiences, providerIdentity, err = - getServiceAccountAndProviderInfo(ctx, provider, o.Client, saRef, opts...) + saInfo, err = getServiceAccountInfo(ctx, provider, o.Client, opts...) if err != nil { return nil, err } // Update the function to create an access token using the service account. - newAccessToken = func() (Token, error) { - // Check the feature gate for object-level workload identity. - if !IsObjectLevelWorkloadIdentityEnabled() { - return nil, ErrObjectLevelWorkloadIdentityNotEnabled + if saInfo.useServiceAccount { + newAccessToken = func() (Token, error) { + // Issue Kubernetes OIDC token for the service account. + tokenReq := &authnv1.TokenRequest{ + Spec: authnv1.TokenRequestSpec{ + Audiences: saInfo.audiences, + }, + } + if err := o.Client.SubResource("token").Create(ctx, saInfo.obj, tokenReq); err != nil { + return nil, fmt.Errorf("failed to create kubernetes token for service account '%s/%s': %w", + saInfo.obj.Namespace, saInfo.obj.Name, err) + } + oidcToken := tokenReq.Status.Token + + // Exchange the Kubernetes OIDC token for a provider access token. + token, err := provider.NewTokenForServiceAccount(ctx, oidcToken, *saInfo.obj, opts...) + if err != nil { + return nil, fmt.Errorf("failed to create provider access token for service account '%s/%s': %w", + saInfo.obj.Namespace, saInfo.obj.Name, err) + } + + return token, nil } + } + } - // Issue Kubernetes OIDC token for the service account. - tokenReq := &authnv1.TokenRequest{ - Spec: authnv1.TokenRequestSpec{ - Audiences: audiences, - }, - } - if err := o.Client.SubResource("token").Create(ctx, serviceAccount, tokenReq); err != nil { - return nil, fmt.Errorf("failed to create kubernetes token for service account '%s/%s': %w", - serviceAccount.Namespace, serviceAccount.Name, err) - } - oidcToken := tokenReq.Status.Token - - // Exchange the Kubernetes OIDC token for a provider access token. - token, err := provider.NewTokenForServiceAccount(ctx, oidcToken, *serviceAccount, opts...) + // Update access token fetcher for impersonation if supported by the provider. + if saInfo != nil && saInfo.providerIdentityForImpersonation != nil { + newNonImpersonatedToken := newAccessToken + newAccessToken = func() (Token, error) { + token, err := newNonImpersonatedToken() if err != nil { - return nil, fmt.Errorf("failed to create provider access token for service account '%s/%s': %w", - serviceAccount.Namespace, serviceAccount.Name, err) + return nil, err } - - return token, nil + p := provider.(ProviderWithImpersonation) + return p.NewTokenForIdentity(ctx, token, saInfo.providerIdentityForImpersonation, opts...) } } @@ -97,7 +102,7 @@ func GetAccessToken(ctx context.Context, provider Provider, opts ...Option) (Tok } // Build cache key. - cacheKey := buildAccessTokenCacheKey(provider, audiences, providerIdentity, serviceAccount, opts...) + cacheKey := buildAccessTokenCacheKey(provider, saInfo, opts...) // Build involved object details. kind := o.InvolvedObject.Kind @@ -116,55 +121,7 @@ func GetAccessToken(ctx context.Context, provider Provider, opts ...Option) (Tok return token, nil } -func getServiceAccountAndProviderInfo(ctx context.Context, provider Provider, client client.Client, - key client.ObjectKey, opts ...Option) (*corev1.ServiceAccount, []string, string, error) { - - var o Options - o.Apply(opts...) - - defaultSA := getDefaultServiceAccount() - var setDefaultSA bool - - // Apply multi-tenancy lockdown: use default service account when .serviceAccountName - // is not explicitly specified in the object. This results in Object-Level Workload Identity. - if key.Name == "" && defaultSA != "" { - key.Name = defaultSA - setDefaultSA = true - } - - // Get service account. - var serviceAccount corev1.ServiceAccount - if err := client.Get(ctx, key, &serviceAccount); err != nil { - if errors.IsNotFound(err) && setDefaultSA { - return nil, nil, "", fmt.Errorf("failed to get service account '%s': %w", - key, ErrDefaultServiceAccountNotFound) - } - return nil, nil, "", fmt.Errorf("failed to get service account '%s': %w", - key, err) - } - - // Get provider audience. - audiences := o.Audiences - if len(audiences) == 0 { - var err error - audiences, err = provider.GetAudiences(ctx, serviceAccount) - if err != nil { - return nil, nil, "", fmt.Errorf("failed to get provider audience: %w", err) - } - } - - // Get provider identity. - providerIdentity, err := provider.GetIdentity(serviceAccount) - if err != nil { - return nil, nil, "", fmt.Errorf("failed to get provider identity from service account '%s/%s' annotations: %w", - key.Namespace, key.Name, err) - } - - return &serviceAccount, audiences, providerIdentity, nil -} - -func buildAccessTokenCacheKey(provider Provider, audiences []string, providerIdentity string, - serviceAccount *corev1.ServiceAccount, opts ...Option) string { +func buildAccessTokenCacheKey(provider Provider, saInfo *serviceAccountInfo, opts ...Option) string { var o Options o.Apply(opts...) @@ -173,11 +130,16 @@ func buildAccessTokenCacheKey(provider Provider, audiences []string, providerIde parts = append(parts, fmt.Sprintf("provider=%s", provider.GetName())) - if serviceAccount != nil { - parts = append(parts, fmt.Sprintf("providerIdentity=%s", providerIdentity)) - parts = append(parts, fmt.Sprintf("serviceAccountName=%s", serviceAccount.Name)) - parts = append(parts, fmt.Sprintf("serviceAccountNamespace=%s", serviceAccount.Namespace)) - parts = append(parts, fmt.Sprintf("serviceAccountTokenAudiences=%s", strings.Join(audiences, ","))) + if saInfo != nil { + if saInfo.useServiceAccount { + parts = append(parts, fmt.Sprintf("serviceAccountName=%s", saInfo.obj.Name)) + parts = append(parts, fmt.Sprintf("serviceAccountNamespace=%s", saInfo.obj.Namespace)) + parts = append(parts, fmt.Sprintf("serviceAccountTokenAudiences=%s", strings.Join(saInfo.audiences, ","))) + parts = append(parts, fmt.Sprintf("providerIdentity=%s", saInfo.providerIdentity)) + } + if saInfo.providerIdentityForImpersonation != nil { + parts = append(parts, fmt.Sprintf("providerIdentityForImpersonation=%s", saInfo.providerIdentityForImpersonation)) + } } if len(o.Scopes) > 0 { diff --git a/auth/access_token_test.go b/auth/access_token_test.go index 21a79b2bd..3a1d6b30e 100644 --- a/auth/access_token_test.go +++ b/auth/access_token_test.go @@ -18,6 +18,7 @@ package auth_test import ( "context" + "fmt" "net/url" "testing" "time" @@ -63,14 +64,81 @@ func TestGetAccessToken(t *testing.T) { err = kubeClient.Create(ctx, lockdownServiceAccount) g.Expect(err).NotTo(HaveOccurred()) + // Create a service account with impersonation annotation. + impersonationServiceAccount := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "impersonation-sa", + Namespace: "default", + Annotations: map[string]string{ + "mock-provider.auth.fluxcd.io/impersonation": "roleArn: arn:aws:iam::123456789012:role/target-role\nuseServiceAccount: true", + }, + }, + } + err = kubeClient.Create(ctx, impersonationServiceAccount) + g.Expect(err).NotTo(HaveOccurred()) + + // Create a service account with impersonation annotation (no useServiceAccount, defaults to false without lockdown). + impersonationNoSAServiceAccount := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "impersonation-no-sa", + Namespace: "default", + Annotations: map[string]string{ + "mock-provider.auth.fluxcd.io/impersonation": "roleArn: arn:aws:iam::123456789012:role/target-role", + }, + }, + } + err = kubeClient.Create(ctx, impersonationNoSAServiceAccount) + g.Expect(err).NotTo(HaveOccurred()) + + // Create a service account with impersonation annotation with explicit useServiceAccount: false. + impersonationExplicitNoSAServiceAccount := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "impersonation-explicit-no-sa", + Namespace: "default", + Annotations: map[string]string{ + "mock-provider.auth.fluxcd.io/impersonation": "roleArn: arn:aws:iam::123456789012:role/target-role\nuseServiceAccount: false", + }, + }, + } + err = kubeClient.Create(ctx, impersonationExplicitNoSAServiceAccount) + g.Expect(err).NotTo(HaveOccurred()) + + // Create a service account with impersonation annotation (useServiceAccount: false) + // AND a provider identity annotation (which should be rejected). + impersonationNoSAWithIdentityServiceAccount := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "impersonation-no-sa-with-identity", + Namespace: "default", + Annotations: map[string]string{ + "mock-provider.auth.fluxcd.io/impersonation": "roleArn: arn:aws:iam::123456789012:role/target-role\nuseServiceAccount: false", + }, + }, + } + err = kubeClient.Create(ctx, impersonationNoSAWithIdentityServiceAccount) + g.Expect(err).NotTo(HaveOccurred()) + + // Create a service account with invalid impersonation annotation. + invalidImpersonationServiceAccount := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "invalid-impersonation-sa", + Namespace: "default", + Annotations: map[string]string{ + "mock-provider.auth.fluxcd.io/impersonation": "{{invalid yaml", + }, + }, + } + err = kubeClient.Create(ctx, invalidImpersonationServiceAccount) + g.Expect(err).NotTo(HaveOccurred()) + for _, tt := range []struct { name string provider *mockProvider + withImpersonation bool opts []auth.Option disableObjectLevel bool - defaultSA string expectedToken auth.Token expectedErr string + verifyIdentity string }{ { name: "controller access token", @@ -116,6 +184,7 @@ func TestGetAccessToken(t *testing.T) { opts: []auth.Option{ auth.WithClient(kubeClient), auth.WithServiceAccountNamespace("default"), + auth.WithDefaultServiceAccount("lockdown-sa"), auth.WithAudiences("audience1", "audience2"), auth.WithScopes("scope1", "scope2"), auth.WithSTSRegion("us-east-1"), @@ -123,7 +192,6 @@ func TestGetAccessToken(t *testing.T) { auth.WithProxyURL(url.URL{Scheme: "http", Host: "proxy.io:8080"}), auth.WithCAData("ca-data"), }, - defaultSA: "lockdown-sa", expectedToken: &mockToken{token: "mock-access-token"}, }, { @@ -138,6 +206,7 @@ func TestGetAccessToken(t *testing.T) { opts: []auth.Option{ auth.WithClient(kubeClient), auth.WithServiceAccountNamespace("default"), + auth.WithDefaultServiceAccount("lockdown-sa"), auth.WithAudiences("audience1", "audience2"), auth.WithScopes("scope1", "scope2"), auth.WithSTSRegion("us-east-1"), @@ -145,7 +214,6 @@ func TestGetAccessToken(t *testing.T) { auth.WithProxyURL(url.URL{Scheme: "http", Host: "proxy.io:8080"}), auth.WithCAData("ca-data"), }, - defaultSA: "lockdown-sa", disableObjectLevel: true, expectedToken: &mockToken{token: "mock-access-token"}, expectedErr: "ObjectLevelWorkloadIdentity feature gate is not enabled", @@ -162,9 +230,9 @@ func TestGetAccessToken(t *testing.T) { opts: []auth.Option{ auth.WithClient(kubeClient), auth.WithServiceAccountNamespace("default"), + auth.WithDefaultServiceAccount("non-existent-sa"), auth.WithAudiences("audience1", "audience2"), }, - defaultSA: "non-existent-sa", expectedErr: "the specified default service account does not exist in the object namespace", }, { @@ -209,6 +277,7 @@ func TestGetAccessToken(t *testing.T) { auth.WithClient(kubeClient), auth.WithServiceAccountName(saRef.Name), auth.WithServiceAccountNamespace(saRef.Namespace), + auth.WithDefaultServiceAccount("non-existent-sa"), auth.WithAudiences("audience1", "audience2"), auth.WithScopes("scope1", "scope2"), auth.WithSTSRegion("us-east-1"), @@ -216,7 +285,6 @@ func TestGetAccessToken(t *testing.T) { auth.WithProxyURL(url.URL{Scheme: "http", Host: "proxy.io:8080"}), auth.WithCAData("ca-data"), }, - defaultSA: "non-existent-sa", expectedToken: &mockToken{token: "mock-access-token"}, }, { @@ -262,7 +330,7 @@ func TestGetAccessToken(t *testing.T) { tokenCache, err := cache.NewTokenCache(1) g.Expect(err).NotTo(HaveOccurred()) - const key = "db625bd5a96dc48fcc100659c6db98857d1e0ceec930bbded0fdece14af4307c" + const key = "6fbdfd364d87e47e6aad554232b927805c949ac461c43eb1c84d7dbcd58c38fb" token := &mockToken{token: "cached-token"} cachedToken, ok, err := tokenCache.GetOrSet(ctx, key, func(ctx context.Context) (cache.Token, error) { return token, nil @@ -293,7 +361,7 @@ func TestGetAccessToken(t *testing.T) { auth.WithProxyURL(url.URL{Scheme: "http", Host: "proxy.io:8080"}), auth.WithCAData("ca-data"), }, - expectedErr: "failed to get provider identity from service account 'default/default' annotations: mock error", + expectedErr: "failed to get provider identity from service account 'default/default' annotations:", }, { name: "error getting identity using cache", @@ -317,7 +385,7 @@ func TestGetAccessToken(t *testing.T) { o.Cache = tokenCache }, }, - expectedErr: "failed to get provider identity from service account 'default/default' annotations: mock error", + expectedErr: "failed to get provider identity from service account 'default/default' annotations:", }, { name: "disable object level workload identity", @@ -338,6 +406,210 @@ func TestGetAccessToken(t *testing.T) { disableObjectLevel: true, expectedErr: "ObjectLevelWorkloadIdentity feature gate is not enabled", }, + { + name: "impersonation skipped without service account", + provider: &mockProvider{ + returnControllerToken: &mockToken{token: "mock-controller-token"}, + returnImpersonatedToken: &mockToken{token: "should-not-appear"}, + }, + withImpersonation: true, + opts: []auth.Option{ + auth.WithAudiences("audience1", "audience2"), + auth.WithScopes("scope1", "scope2"), + auth.WithSTSRegion("us-east-1"), + auth.WithSTSEndpoint("https://sts.some-cloud.io"), + auth.WithProxyURL(url.URL{Scheme: "http", Host: "proxy.io:8080"}), + auth.WithCAData("ca-data"), + }, + expectedToken: &mockToken{token: "mock-controller-token"}, + }, + { + name: "access token from service account with impersonation", + provider: &mockProvider{ + returnName: "mock-provider", + returnAccessToken: &mockToken{token: "mock-access-token"}, + returnIdentityForImpersonation: mockIdentity("arn:aws:iam::123456789012:role/target-role"), + returnImpersonatedToken: &mockToken{token: "mock-impersonated-sa-token"}, + paramAudiences: []string{"audience1", "audience2"}, + paramServiceAccount: *impersonationServiceAccount, + paramOIDCTokenClient: oidcClient, + }, + withImpersonation: true, + opts: []auth.Option{ + auth.WithClient(kubeClient), + auth.WithServiceAccountName("impersonation-sa"), + auth.WithServiceAccountNamespace("default"), + auth.WithAudiences("audience1", "audience2"), + auth.WithScopes("scope1", "scope2"), + auth.WithSTSRegion("us-east-1"), + auth.WithSTSEndpoint("https://sts.some-cloud.io"), + auth.WithProxyURL(url.URL{Scheme: "http", Host: "proxy.io:8080"}), + auth.WithCAData("ca-data"), + }, + expectedToken: &mockToken{token: "mock-impersonated-sa-token"}, + verifyIdentity: "arn:aws:iam::123456789012:role/target-role", + }, + { + name: "impersonation with useServiceAccount false uses controller token", + provider: &mockProvider{ + returnName: "mock-provider", + returnControllerToken: &mockToken{token: "mock-controller-token"}, + returnIdentityForImpersonation: mockIdentity("arn:aws:iam::123456789012:role/target-role"), + returnImpersonatedToken: &mockToken{token: "mock-impersonated-controller-token"}, + paramServiceAccount: *impersonationNoSAServiceAccount, + }, + withImpersonation: true, + opts: []auth.Option{ + auth.WithClient(kubeClient), + auth.WithServiceAccountName("impersonation-no-sa"), + auth.WithServiceAccountNamespace("default"), + auth.WithAudiences("audience1", "audience2"), + auth.WithScopes("scope1", "scope2"), + auth.WithSTSRegion("us-east-1"), + auth.WithSTSEndpoint("https://sts.some-cloud.io"), + auth.WithProxyURL(url.URL{Scheme: "http", Host: "proxy.io:8080"}), + auth.WithCAData("ca-data"), + }, + expectedToken: &mockToken{token: "mock-impersonated-controller-token"}, + verifyIdentity: "arn:aws:iam::123456789012:role/target-role", + }, + { + name: "impersonation with useServiceAccount false and lockdown enabled errors", + provider: &mockProvider{ + returnName: "mock-provider", + returnIdentityForImpersonation: mockIdentity("arn:aws:iam::123456789012:role/target-role"), + paramServiceAccount: *impersonationExplicitNoSAServiceAccount, + }, + withImpersonation: true, + opts: []auth.Option{ + auth.WithClient(kubeClient), + auth.WithServiceAccountNamespace("default"), + auth.WithDefaultServiceAccount("impersonation-explicit-no-sa"), + auth.WithAudiences("audience1", "audience2"), + auth.WithScopes("scope1", "scope2"), + auth.WithSTSRegion("us-east-1"), + auth.WithSTSEndpoint("https://sts.some-cloud.io"), + auth.WithProxyURL(url.URL{Scheme: "http", Host: "proxy.io:8080"}), + auth.WithCAData("ca-data"), + }, + expectedErr: "multi-tenancy lockdown is enabled, impersonation without service account is not allowed", + }, + { + name: "impersonation NewTokenForIdentity error", + provider: &mockProvider{ + returnName: "mock-provider", + returnAccessToken: &mockToken{token: "mock-access-token"}, + returnIdentityForImpersonation: mockIdentity("arn:aws:iam::123456789012:role/target-role"), + returnImpersonateErr: fmt.Errorf("impersonation failed"), + paramAudiences: []string{"audience1", "audience2"}, + paramServiceAccount: *impersonationServiceAccount, + paramOIDCTokenClient: oidcClient, + }, + withImpersonation: true, + opts: []auth.Option{ + auth.WithClient(kubeClient), + auth.WithServiceAccountName("impersonation-sa"), + auth.WithServiceAccountNamespace("default"), + auth.WithAudiences("audience1", "audience2"), + auth.WithScopes("scope1", "scope2"), + auth.WithSTSRegion("us-east-1"), + auth.WithSTSEndpoint("https://sts.some-cloud.io"), + auth.WithProxyURL(url.URL{Scheme: "http", Host: "proxy.io:8080"}), + auth.WithCAData("ca-data"), + }, + expectedErr: "impersonation failed", + verifyIdentity: "arn:aws:iam::123456789012:role/target-role", + }, + { + name: "impersonation skipped when no impersonation annotation", + provider: &mockProvider{ + returnName: "mock-provider", + returnAccessToken: &mockToken{token: "mock-access-token"}, + returnImpersonatedToken: &mockToken{token: "should-not-appear"}, + paramAudiences: []string{"audience1", "audience2"}, + paramServiceAccount: *defaultServiceAccount, + paramOIDCTokenClient: oidcClient, + }, + withImpersonation: true, + opts: []auth.Option{ + auth.WithClient(kubeClient), + auth.WithServiceAccountName(saRef.Name), + auth.WithServiceAccountNamespace(saRef.Namespace), + auth.WithAudiences("audience1", "audience2"), + auth.WithScopes("scope1", "scope2"), + auth.WithSTSRegion("us-east-1"), + auth.WithSTSEndpoint("https://sts.some-cloud.io"), + auth.WithProxyURL(url.URL{Scheme: "http", Host: "proxy.io:8080"}), + auth.WithCAData("ca-data"), + }, + expectedToken: &mockToken{token: "mock-access-token"}, + }, + { + name: "GetIdentityForImpersonation error", + provider: &mockProvider{ + returnName: "mock-provider", + returnAccessToken: &mockToken{token: "mock-access-token"}, + returnIdentityForImpersonationErr: "impersonation identity lookup failed", + paramAudiences: []string{"audience1", "audience2"}, + paramServiceAccount: *impersonationServiceAccount, + paramOIDCTokenClient: oidcClient, + }, + withImpersonation: true, + opts: []auth.Option{ + auth.WithClient(kubeClient), + auth.WithServiceAccountName("impersonation-sa"), + auth.WithServiceAccountNamespace("default"), + auth.WithAudiences("audience1", "audience2"), + auth.WithScopes("scope1", "scope2"), + auth.WithSTSRegion("us-east-1"), + auth.WithSTSEndpoint("https://sts.some-cloud.io"), + auth.WithProxyURL(url.URL{Scheme: "http", Host: "proxy.io:8080"}), + auth.WithCAData("ca-data"), + }, + expectedErr: "failed to get provider identity for impersonation from service account 'default/impersonation-sa'", + }, + { + name: "invalid impersonation annotation YAML", + provider: &mockProvider{ + returnName: "mock-provider", + paramServiceAccount: *invalidImpersonationServiceAccount, + }, + withImpersonation: true, + opts: []auth.Option{ + auth.WithClient(kubeClient), + auth.WithServiceAccountName("invalid-impersonation-sa"), + auth.WithServiceAccountNamespace("default"), + auth.WithAudiences("audience1", "audience2"), + auth.WithScopes("scope1", "scope2"), + auth.WithSTSRegion("us-east-1"), + auth.WithSTSEndpoint("https://sts.some-cloud.io"), + auth.WithProxyURL(url.URL{Scheme: "http", Host: "proxy.io:8080"}), + auth.WithCAData("ca-data"), + }, + expectedErr: "failed to parse impersonation annotation", + }, + { + name: "impersonation useServiceAccount false with identity annotation errors", + provider: &mockProvider{ + returnName: "mock-provider", + returnIdentity: "mock-identity", + returnIdentityForImpersonation: mockIdentity("arn:aws:iam::123456789012:role/target-role"), + paramServiceAccount: *impersonationNoSAWithIdentityServiceAccount, + }, + withImpersonation: true, + opts: []auth.Option{ + auth.WithClient(kubeClient), + auth.WithServiceAccountName("impersonation-no-sa-with-identity"), + auth.WithServiceAccountNamespace("default"), + auth.WithAudiences("audience1", "audience2"), + auth.WithScopes("scope1", "scope2"), + auth.WithSTSRegion("us-east-1"), + auth.WithSTSEndpoint("https://sts.some-cloud.io"), + auth.WithProxyURL(url.URL{Scheme: "http", Host: "proxy.io:8080"}), + auth.WithCAData("ca-data"), + }, + expectedErr: "identity annotation is present but the ServiceAccount is not used", + }, } { t.Run(tt.name, func(t *testing.T) { g := NewWithT(t) @@ -349,12 +621,12 @@ func TestGetAccessToken(t *testing.T) { t.Cleanup(auth.DisableObjectLevelWorkloadIdentity) } - if tt.defaultSA != "" { - auth.SetDefaultServiceAccount(tt.defaultSA) - t.Cleanup(func() { auth.SetDefaultServiceAccount("") }) + var p auth.Provider = tt.provider + if tt.withImpersonation { + p = &mockProviderWithImpersonation{mockProvider: tt.provider} } - token, err := auth.GetAccessToken(ctx, tt.provider, tt.opts...) + token, err := auth.GetAccessToken(ctx, p, tt.opts...) if tt.expectedErr != "" { g.Expect(err).To(MatchError(ContainSubstring(tt.expectedErr))) @@ -363,6 +635,11 @@ func TestGetAccessToken(t *testing.T) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(token).To(Equal(tt.expectedToken)) } + + if tt.verifyIdentity != "" { + g.Expect(tt.provider.gotIdentity).NotTo(BeNil()) + g.Expect(tt.provider.gotIdentity.String()).To(Equal(tt.verifyIdentity)) + } }) } } diff --git a/auth/api.go b/auth/api.go new file mode 100644 index 000000000..b38d3762e --- /dev/null +++ b/auth/api.go @@ -0,0 +1,79 @@ +/* +Copyright 2026 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "encoding/json" + "fmt" +) + +const ( + // APIGroup is the API group for auth-related APIs, e.g. annotations on ServiceAccounts. + APIGroup = "auth.fluxcd.io" +) + +// ImpersonationAnnotation is the annotation key that should be used in the +// ServiceAccount to specify the impersonation configuration. +func ImpersonationAnnotation(provider ProviderWithImpersonation) string { + return fmt.Sprintf("%s.%s/%s", + provider.GetName(), + APIGroup, + provider.GetImpersonationAnnotationKey()) +} + +// Impersonation contains the cloud provider identity that should be impersonated +// after acquiring an initial cloud provider access token. +type Impersonation struct { + // Identity contains the unmarshaled text used for creating an instance of this + // struct. This allows providers to define their own fields for expressing an + // identity. + Identity json.RawMessage `json:"-"` + + // UseServiceAccount indicates whether OIDC exchange using a token issued for the + // ServiceAccount should be used to get the initial cloud provider access token, + // before impersonating Identity. This field exists to support providers that do + // not require an initial identity to get the initial access token, e.g. GCP does + // not require the Kubernetes ServiceAccount to be associated with a GCP service + // account. Because of this property, it's not possible to decide whether to use + // the ServiceAccount for the initial token exchange or not by just looking at + // the presence of provider-specific annotations on the ServiceAccount indicating + // the initial identity. In other words, any combinations of this field and the + // ServiceAccount annotation for the initial identity are valid for GCP. Setting + // this field to false on multi-tenancy lockdown is not a valid configuration and + // will result in an error. When multi-tenancy lockdown is not enabled, the default + // value of this field is false, which means that the initial cloud provider access + // token will be retrieved from the environment of the controller pod, e.g. from + // files mounted in the pod, environment variables, local metadata services, etc. + // When multi-tenancy lockdown is enabled, the default value of this field is true, + // which means that the ServiceAccount token will be used for the initial token + // exchange. + UseServiceAccount *bool `json:"useServiceAccount,omitempty"` +} + +// UnmarshalJSON implements the json.Unmarshaler interface to allow the custom unmarshaling +// logic for the Impersonation struct as described in the field comments. +func (i *Impersonation) UnmarshalJSON(data []byte) error { + var aux struct { + UseServiceAccount *bool `json:"useServiceAccount,omitempty"` + } + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + i.Identity = json.RawMessage(data) + i.UseServiceAccount = aux.UseServiceAccount + return nil +} diff --git a/auth/aws/implementation.go b/auth/aws/implementation.go index 98e79a669..0025786e5 100644 --- a/auth/aws/implementation.go +++ b/auth/aws/implementation.go @@ -32,6 +32,7 @@ import ( type Implementation interface { LoadDefaultConfig(ctx context.Context, optFns ...func(*config.LoadOptions) error) (aws.Config, error) AssumeRoleWithWebIdentity(ctx context.Context, params *sts.AssumeRoleWithWebIdentityInput, options sts.Options) (*sts.AssumeRoleWithWebIdentityOutput, error) + AssumeRole(ctx context.Context, params *sts.AssumeRoleInput, options sts.Options) (*sts.AssumeRoleOutput, error) GetAuthorizationToken(ctx context.Context, cfg aws.Config) (any, error) GetPublicAuthorizationToken(ctx context.Context, cfg aws.Config) (any, error) DescribeCluster(ctx context.Context, params *eks.DescribeClusterInput, options eks.Options) (*eks.DescribeClusterOutput, error) @@ -48,6 +49,10 @@ func (implementation) AssumeRoleWithWebIdentity(ctx context.Context, params *sts return sts.New(options).AssumeRoleWithWebIdentity(ctx, params) } +func (implementation) AssumeRole(ctx context.Context, params *sts.AssumeRoleInput, options sts.Options) (*sts.AssumeRoleOutput, error) { + return sts.New(options).AssumeRole(ctx, params) +} + func (implementation) GetAuthorizationToken(ctx context.Context, cfg aws.Config) (any, error) { return ecr.NewFromConfig(cfg).GetAuthorizationToken(ctx, &ecr.GetAuthorizationTokenInput{}) } diff --git a/auth/aws/implementation_test.go b/auth/aws/implementation_test.go index f9523df84..5cac63100 100644 --- a/auth/aws/implementation_test.go +++ b/auth/aws/implementation_test.go @@ -61,6 +61,13 @@ type mockImplementation struct { returnEndpoint string returnCAData string returnPresignedURL string + + // AssumeRole fields + argAssumeRoleARN string + argAssumeRoleSessionName string + argAssumeRoleCredsProvider aws.CredentialsProvider + returnAssumeRoleCreds aws.Credentials + returnAssumeRoleErr error } type mockHTTPPresigner struct { @@ -122,6 +129,43 @@ func (m *mockImplementation) AssumeRoleWithWebIdentity(ctx context.Context, para }, nil } +func (m *mockImplementation) AssumeRole(ctx context.Context, params *sts.AssumeRoleInput, options sts.Options) (*sts.AssumeRoleOutput, error) { + m.t.Helper() + g := NewWithT(m.t) + g.Expect(params).NotTo(BeNil()) + g.Expect(params.RoleArn).NotTo(BeNil()) + g.Expect(*params.RoleArn).To(Equal(m.argAssumeRoleARN)) + g.Expect(params.RoleSessionName).NotTo(BeNil()) + g.Expect(*params.RoleSessionName).To(Equal(m.argAssumeRoleSessionName)) + g.Expect(options.Region).To(Equal(m.argRegion)) + if m.argAssumeRoleCredsProvider != nil { + g.Expect(options.Credentials).To(Equal(m.argAssumeRoleCredsProvider)) + } + if m.argSTSEndpoint != "" { + g.Expect(options.BaseEndpoint).NotTo(BeNil()) + g.Expect(*options.BaseEndpoint).To(Equal(m.argSTSEndpoint)) + } + g.Expect(options.HTTPClient).NotTo(BeNil()) + g.Expect(options.HTTPClient.(*http.Client)).NotTo(BeNil()) + g.Expect(options.HTTPClient.(*http.Client).Transport).NotTo(BeNil()) + g.Expect(options.HTTPClient.(*http.Client).Transport.(*http.Transport)).NotTo(BeNil()) + g.Expect(options.HTTPClient.(*http.Client).Transport.(*http.Transport).Proxy).NotTo(BeNil()) + proxyURL, err := options.HTTPClient.(*http.Client).Transport.(*http.Transport).Proxy(nil) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(proxyURL).To(Equal(m.argProxyURL)) + if m.returnAssumeRoleErr != nil { + return nil, m.returnAssumeRoleErr + } + return &sts.AssumeRoleOutput{ + Credentials: &ststypes.Credentials{ + AccessKeyId: aws.String(m.returnAssumeRoleCreds.AccessKeyID), + SecretAccessKey: aws.String(m.returnAssumeRoleCreds.SecretAccessKey), + SessionToken: aws.String(m.returnAssumeRoleCreds.SessionToken), + Expiration: aws.Time(m.returnAssumeRoleCreds.Expires), + }, + }, nil +} + func (m *mockImplementation) GetAuthorizationToken(ctx context.Context, cfg aws.Config) (any, error) { m.t.Helper() g := NewWithT(m.t) diff --git a/auth/aws/options.go b/auth/aws/options.go index 7c7a0bf8d..3d518c326 100644 --- a/auth/aws/options.go +++ b/auth/aws/options.go @@ -17,10 +17,13 @@ limitations under the License. package aws import ( + "errors" "fmt" "regexp" corev1 "k8s.io/api/core/v1" + + "github.com/fluxcd/pkg/auth" ) const stsEndpointPattern = `^https://(.+\.)?sts(-fips)?(\.[^.]+)?(\.vpce)?\.amazonaws\.com$` @@ -45,7 +48,7 @@ func ValidateSTSEndpoint(endpoint string) error { return nil } -const roleARNPattern = `^arn:aws[\w-]*:iam::[0-9]{1,30}:role/.{1,200}$` +const roleARNPattern = `^arn:aws[\w-]*:iam::[0-9]{1,30}:role/(.{1,200})$` var roleARNRegex = regexp.MustCompile(roleARNPattern) @@ -59,12 +62,25 @@ func getRoleARN(serviceAccount corev1.ServiceAccount) (string, error) { return arn, nil } -func getRoleSessionName(serviceAccount corev1.ServiceAccount, region string) string { +func getRoleNameFromARN(arn string) (string, error) { + m := roleARNRegex.FindStringSubmatch(arn) + if len(m) != 2 { + return "", fmt.Errorf("invalid role ARN: '%s'. must match %s", + arn, roleARNPattern) + } + return m[1], nil +} + +func getRoleSessionNameForServiceAccount(serviceAccount corev1.ServiceAccount, region string) string { name := serviceAccount.Name namespace := serviceAccount.Namespace return fmt.Sprintf("%s.%s.%s.fluxcd.io", name, namespace, region) } +func getRoleSessionNameForImpersonation(roleName, region string) string { + return fmt.Sprintf("%s.%s.fluxcd.io", roleName, region) +} + const clusterPattern = `^arn:aws[\w-]*:eks:([^:]{1,100}):[0-9]{1,30}:cluster/(.{1,200})$` var clusterRegex = regexp.MustCompile(clusterPattern) @@ -79,3 +95,18 @@ func parseCluster(cluster string) (string, string, error) { name := m[2] return region, name, nil } + +func getSTSRegionForObjectLevel(o *auth.Options) (string, error) { + stsRegion := o.STSRegion + if stsRegion == "" { + // In this case we can't rely on IRSA or EKS Pod Identity for the controller + // pod because this is object-level configuration, so we show a different + // error message. + // In this error message we assume an API that has a region field, e.g. the + // Bucket API. APIs that can extract the region from the ARN (e.g. KMS) will + // never reach this code path. + return "", errors.New("an AWS region is required for authenticating with a service account. " + + "please configure one in the object spec") + } + return stsRegion, nil +} diff --git a/auth/aws/provider.go b/auth/aws/provider.go index 9cec5c6b3..41cd97e51 100644 --- a/auth/aws/provider.go +++ b/auth/aws/provider.go @@ -19,6 +19,7 @@ package aws import ( "context" "encoding/base64" + "encoding/json" "errors" "fmt" "os" @@ -98,11 +99,7 @@ func (Provider) GetAudiences(context.Context, corev1.ServiceAccount) ([]string, // GetIdentity implements auth.Provider. func (Provider) GetIdentity(serviceAccount corev1.ServiceAccount) (string, error) { - roleARN, err := getRoleARN(serviceAccount) - if err != nil { - return "", err - } - return roleARN, nil + return getRoleARN(serviceAccount) } // NewTokenForServiceAccount implements auth.Provider. @@ -112,16 +109,9 @@ func (p Provider) NewTokenForServiceAccount(ctx context.Context, oidcToken strin var o auth.Options o.Apply(opts...) - stsRegion := o.STSRegion - if stsRegion == "" { - // In this case we can't rely on IRSA or EKS Pod Identity for the controller - // pod because this is object-level configuration, so we show a different - // error message. - // In this error message we assume an API that has a region field, e.g. the - // Bucket API. APIs that can extract the region from the ARN (e.g. KMS) will - // never reach this code path. - return nil, errors.New("an AWS region is required for authenticating with a service account. " + - "please configure one in the object spec") + stsRegion, err := getSTSRegionForObjectLevel(&o) + if err != nil { + return nil, err } roleARN, err := getRoleARN(serviceAccount) @@ -129,7 +119,7 @@ func (p Provider) NewTokenForServiceAccount(ctx context.Context, oidcToken strin return nil, err } - roleSessionName := getRoleSessionName(serviceAccount, stsRegion) + roleSessionName := getRoleSessionNameForServiceAccount(serviceAccount, stsRegion) stsOpts := sts.Options{ Region: stsRegion, @@ -164,6 +154,85 @@ func (p Provider) NewTokenForServiceAccount(ctx context.Context, oidcToken strin return creds, nil } +// GetImpersonationAnnotationKey implements auth.ProviderWithImpersonation. +func (Provider) GetImpersonationAnnotationKey() string { + return "assume-role" +} + +type impersonation struct { + RoleARN string `json:"roleARN"` +} + +func (i impersonation) String() string { + return i.RoleARN +} + +// GetIdentityForImpersonation implements auth.ProviderWithImpersonation. +func (Provider) GetIdentityForImpersonation(identity json.RawMessage) (fmt.Stringer, error) { + var id impersonation + if err := json.Unmarshal(identity, &id); err != nil { + return nil, fmt.Errorf("failed to unmarshal impersonation identity: %w", err) + } + if !roleARNRegex.MatchString(id.RoleARN) { + return nil, fmt.Errorf("invalid role ARN in impersonation identity: '%s'. must match %s", + id.RoleARN, roleARNPattern) + } + return &id, nil +} + +// NewTokenForIdentity implements auth.ProviderWithImpersonation. +func (p Provider) NewTokenForIdentity(ctx context.Context, token auth.Token, + identity fmt.Stringer, opts ...auth.Option) (auth.Token, error) { + + var o auth.Options + o.Apply(opts...) + + stsRegion, err := getSTSRegionForObjectLevel(&o) + if err != nil { + return nil, err + } + + roleARN := identity.(*impersonation).RoleARN + + roleName, err := getRoleNameFromARN(roleARN) + if err != nil { + return nil, err + } + roleSessionName := getRoleSessionNameForImpersonation(roleName, stsRegion) + + stsOpts := sts.Options{ + Region: stsRegion, + Credentials: token.(*Credentials).provider(), + HTTPClient: o.GetHTTPClient(), + } + + if e := o.STSEndpoint; e != "" { + if err := ValidateSTSEndpoint(e); err != nil { + return nil, err + } + stsOpts.BaseEndpoint = &e + } + + req := &sts.AssumeRoleInput{ + RoleArn: &roleARN, + RoleSessionName: &roleSessionName, + } + resp, err := p.impl().AssumeRole(ctx, req, stsOpts) + if err != nil { + return nil, err + } + if resp.Credentials == nil { + return nil, fmt.Errorf("credentials are nil") + } + + creds := &Credentials{*resp.Credentials} + if creds.Expiration == nil { + creds.Expiration = &time.Time{} + } + + return creds, nil +} + // GetAccessTokenOptionsForArtifactRepository implements auth.Provider. func (p Provider) GetAccessTokenOptionsForArtifactRepository(artifactRepository string) ([]auth.Option, error) { // AWS requires a region for getting access credentials. To avoid requiring diff --git a/auth/aws/provider_test.go b/auth/aws/provider_test.go index 92b928f0e..1f9f5c45b 100644 --- a/auth/aws/provider_test.go +++ b/auth/aws/provider_test.go @@ -138,7 +138,7 @@ func TestProvider_NewTokenForServiceAccount(t *testing.T) { name: "invalid role ARN", roleARN: "foobar", stsEndpoint: "https://sts.amazonaws.com", - err: `invalid eks.amazonaws.com/role-arn annotation: 'foobar'. must match ^arn:aws[\w-]*:iam::[0-9]{1,30}:role/.{1,200}$`, + err: `invalid eks.amazonaws.com/role-arn annotation: 'foobar'. must match ^arn:aws[\w-]*:iam::[0-9]{1,30}:role/(.{1,200})$`, }, } { t.Run(tt.name, func(t *testing.T) { @@ -214,6 +214,149 @@ func TestProvider_GetIdentity(t *testing.T) { g.Expect(identity).To(Equal("arn:aws:iam::1234567890:role/some-role")) } +func TestProvider_GetImpersonationAnnotationKey(t *testing.T) { + g := NewWithT(t) + g.Expect(aws.Provider{}.GetImpersonationAnnotationKey()).To(Equal("assume-role")) +} + +func TestProvider_GetIdentityForImpersonation(t *testing.T) { + for _, tt := range []struct { + name string + identity string + expected string + err string + }{ + { + name: "valid role ARN", + identity: `{"roleARN":"arn:aws:iam::123456789012:role/some-role"}`, + expected: "arn:aws:iam::123456789012:role/some-role", + }, + { + name: "valid us-gov role ARN", + identity: `{"roleARN":"arn:aws-us-gov:iam::123456789012:role/some-role"}`, + expected: "arn:aws-us-gov:iam::123456789012:role/some-role", + }, + { + name: "invalid role ARN", + identity: `{"roleARN":"foobar"}`, + err: "invalid role ARN in impersonation identity: 'foobar'", + }, + { + name: "empty role ARN", + identity: `{"roleARN":""}`, + err: "invalid role ARN in impersonation identity: ''", + }, + { + name: "invalid JSON", + identity: `{invalid`, + err: "failed to unmarshal impersonation identity", + }, + } { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + identity, err := aws.Provider{}.GetIdentityForImpersonation([]byte(tt.identity)) + + if tt.err != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.err)) + g.Expect(identity).To(BeNil()) + } else { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(identity).NotTo(BeNil()) + g.Expect(identity.String()).To(Equal(tt.expected)) + } + }) + } +} + +func TestProvider_NewTokenForIdentity(t *testing.T) { + for _, tt := range []struct { + name string + roleARN string + stsEndpoint string + skipRegion bool + err string + }{ + { + name: "valid", + roleARN: "arn:aws:iam::123456789012:role/target-role", + stsEndpoint: "https://sts.amazonaws.com", + }, + { + name: "us gov role ARN", + roleARN: "arn:aws-us-gov:iam::123456789012:role/target-role", + stsEndpoint: "https://sts.amazonaws.com", + }, + { + name: "missing region", + roleARN: "arn:aws:iam::123456789012:role/target-role", + stsEndpoint: "https://sts.amazonaws.com", + skipRegion: true, + err: "an AWS region is required for authenticating with a service account. " + + "please configure one in the object spec", + }, + { + name: "invalid STS endpoint", + roleARN: "arn:aws:iam::123456789012:role/target-role", + stsEndpoint: "https://something.amazonaws.com", + err: `invalid STS endpoint: 'https://something.amazonaws.com'`, + }, + } { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + impl := &mockImplementation{ + t: t, + argRegion: "us-east-1", + argAssumeRoleARN: tt.roleARN, + argAssumeRoleSessionName: "target-role.us-east-1.fluxcd.io", + argAssumeRoleCredsProvider: credentials.NewStaticCredentialsProvider("initial-key-id", "initial-secret", "initial-session"), + argProxyURL: &url.URL{Scheme: "http", Host: "proxy.example.com"}, + argSTSEndpoint: tt.stsEndpoint, + returnAssumeRoleCreds: awssdk.Credentials{AccessKeyID: "assumed-key-id"}, + } + + // Create the identity via GetIdentityForImpersonation. + identity, err := aws.Provider{}.GetIdentityForImpersonation( + []byte(`{"roleARN":"` + tt.roleARN + `"}`)) + g.Expect(err).NotTo(HaveOccurred()) + + // Create a mock initial token. + initialToken := &aws.Credentials{Credentials: types.Credentials{ + AccessKeyId: awssdk.String("initial-key-id"), + SecretAccessKey: awssdk.String("initial-secret"), + SessionToken: awssdk.String("initial-session"), + }} + + opts := []auth.Option{ + auth.WithProxyURL(url.URL{Scheme: "http", Host: "proxy.example.com"}), + auth.WithSTSEndpoint(tt.stsEndpoint), + } + if !tt.skipRegion { + opts = append(opts, auth.WithSTSRegion("us-east-1")) + } + + provider := aws.Provider{Implementation: impl} + token, err := provider.NewTokenForIdentity(context.Background(), initialToken, identity, opts...) + + if tt.err != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.err)) + g.Expect(token).To(BeNil()) + } else { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(token).To(Equal(&aws.Credentials{Credentials: types.Credentials{ + AccessKeyId: awssdk.String("assumed-key-id"), + SecretAccessKey: awssdk.String(""), + SessionToken: awssdk.String(""), + Expiration: awssdk.Time(time.Time{}), + }})) + } + }) + } +} + func TestProvider_NewArtifactRegistryCredentials(t *testing.T) { for _, tt := range []struct { name string diff --git a/auth/controller_flags.go b/auth/controller_flags.go index fa44a0730..a2ed6ec05 100644 --- a/auth/controller_flags.go +++ b/auth/controller_flags.go @@ -36,61 +36,9 @@ const ( ControllerFlagDefaultDecryptionServiceAccount = "default-decryption-service-account" ) -var ( - // defaultServiceAccount stores the default service account name - // for workload identity. - defaultServiceAccount string - - // defaultKubeConfigServiceAccount stores the default kubeconfig - // service account name. - defaultKubeConfigServiceAccount string - - // defaultDecryptionServiceAccount stores the default decryption - // service account name. - defaultDecryptionServiceAccount string -) - // ErrDefaultServiceAccountNotFound is returned when a default service account // configured by the operator is not found in the user's namespace. -var ErrDefaultServiceAccountNotFound = fmt.Errorf("the specified default service account does not exist in the object namespace. your cluster is subject to multi-tenant workload identity lockdown, reach out to your cluster administrator for help") - -// SetDefaultServiceAccount sets the default service account name for workload identity. -func SetDefaultServiceAccount(sa string) { - defaultServiceAccount = sa -} - -// SetDefaultKubeConfigServiceAccount sets the default kubeconfig service account name. -func SetDefaultKubeConfigServiceAccount(sa string) { - defaultKubeConfigServiceAccount = sa -} - -// SetDefaultDecryptionServiceAccount sets the default decryption service account name. -func SetDefaultDecryptionServiceAccount(sa string) { - defaultDecryptionServiceAccount = sa -} - -// GetDefaultServiceAccount returns the default service account name for workload identity. -func GetDefaultServiceAccount() string { - return defaultServiceAccount -} - -// GetDefaultKubeConfigServiceAccount returns the default kubeconfig service account name. -func GetDefaultKubeConfigServiceAccount() string { - return defaultKubeConfigServiceAccount -} - -// GetDefaultDecryptionServiceAccount returns the default decryption service account name. -func GetDefaultDecryptionServiceAccount() string { - return defaultDecryptionServiceAccount -} - -func getDefaultServiceAccount() string { - // Here we can detect a default service account by checking either the default - // service account or the default kubeconfig service account because these two - // are supposed to never be set simultaneously. The controller main functions - // must ensure this property. - if s := GetDefaultServiceAccount(); s != "" { - return s - } - return GetDefaultKubeConfigServiceAccount() -} +var ErrDefaultServiceAccountNotFound = fmt.Errorf( + "the specified default service account does not exist in the object namespace. " + + "your cluster is subject to multi-tenant workload identity lockdown, reach out " + + "to your cluster administrator for help") diff --git a/auth/controller_flags_test.go b/auth/controller_flags_test.go deleted file mode 100644 index 05b44a214..000000000 --- a/auth/controller_flags_test.go +++ /dev/null @@ -1,103 +0,0 @@ -/* -Copyright 2025 The Flux authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package auth_test - -import ( - "testing" - - . "github.com/onsi/gomega" - - "github.com/fluxcd/pkg/auth" -) - -func TestSetDefaultServiceAccount(t *testing.T) { - g := NewWithT(t) - - auth.SetDefaultServiceAccount("test-sa") - t.Cleanup(func() { auth.SetDefaultServiceAccount("") }) - - g.Expect(auth.GetDefaultServiceAccount()).To(Equal("test-sa")) -} - -func TestSetDefaultKubeConfigServiceAccount(t *testing.T) { - g := NewWithT(t) - - auth.SetDefaultKubeConfigServiceAccount("test-kubeconfig-sa") - t.Cleanup(func() { auth.SetDefaultKubeConfigServiceAccount("") }) - - g.Expect(auth.GetDefaultKubeConfigServiceAccount()).To(Equal("test-kubeconfig-sa")) -} - -func TestSetDefaultDecryptionServiceAccount(t *testing.T) { - g := NewWithT(t) - - auth.SetDefaultDecryptionServiceAccount("test-decryption-sa") - t.Cleanup(func() { auth.SetDefaultDecryptionServiceAccount("") }) - - g.Expect(auth.GetDefaultDecryptionServiceAccount()).To(Equal("test-decryption-sa")) -} - -func TestGetDefaultServiceAccount(t *testing.T) { - t.Run("returns set value", func(t *testing.T) { - g := NewWithT(t) - - auth.SetDefaultServiceAccount("expected-sa") - t.Cleanup(func() { auth.SetDefaultServiceAccount("") }) - - g.Expect(auth.GetDefaultServiceAccount()).To(Equal("expected-sa")) - }) - - t.Run("returns empty when not set", func(t *testing.T) { - g := NewWithT(t) - - g.Expect(auth.GetDefaultServiceAccount()).To(Equal("")) - }) -} - -func TestGetDefaultKubeConfigServiceAccount(t *testing.T) { - t.Run("returns set value", func(t *testing.T) { - g := NewWithT(t) - - auth.SetDefaultKubeConfigServiceAccount("expected-kubeconfig-sa") - t.Cleanup(func() { auth.SetDefaultKubeConfigServiceAccount("") }) - - g.Expect(auth.GetDefaultKubeConfigServiceAccount()).To(Equal("expected-kubeconfig-sa")) - }) - - t.Run("returns empty when not set", func(t *testing.T) { - g := NewWithT(t) - - g.Expect(auth.GetDefaultKubeConfigServiceAccount()).To(Equal("")) - }) -} - -func TestGetDefaultDecryptionServiceAccount(t *testing.T) { - t.Run("returns set value", func(t *testing.T) { - g := NewWithT(t) - - auth.SetDefaultDecryptionServiceAccount("expected-decryption-sa") - t.Cleanup(func() { auth.SetDefaultDecryptionServiceAccount("") }) - - g.Expect(auth.GetDefaultDecryptionServiceAccount()).To(Equal("expected-decryption-sa")) - }) - - t.Run("returns empty when not set", func(t *testing.T) { - g := NewWithT(t) - - g.Expect(auth.GetDefaultDecryptionServiceAccount()).To(Equal("")) - }) -} diff --git a/auth/controller_options.go b/auth/controller_options.go index 03e46ddce..caf23c2ea 100644 --- a/auth/controller_options.go +++ b/auth/controller_options.go @@ -27,10 +27,17 @@ var ErrInconsistentObjectLevelConfiguration = fmt.Errorf( FeatureGateObjectLevelWorkloadIdentity) // InconsistentObjectLevelConfiguration checks if the controller's object-level -// workload identity configuration is inconsistent. -func InconsistentObjectLevelConfiguration() bool { - return !IsObjectLevelWorkloadIdentityEnabled() && - (GetDefaultServiceAccount() != "" || - GetDefaultKubeConfigServiceAccount() != "" || - GetDefaultDecryptionServiceAccount() != "") +// workload identity configuration is inconsistent, which is the case when the +// ObjectLevelWorkloadIdentity feature gate is not enabled but default service +// accounts are set. +func InconsistentObjectLevelConfiguration(defaultServiceAccounts ...string) bool { + if IsObjectLevelWorkloadIdentityEnabled() { + return false + } + for _, sa := range defaultServiceAccounts { + if sa != "" { + return true + } + } + return false } diff --git a/auth/controller_options_test.go b/auth/controller_options_test.go index 17e5046e9..bf53dc051 100644 --- a/auth/controller_options_test.go +++ b/auth/controller_options_test.go @@ -95,20 +95,14 @@ func TestInconsistentObjectLevelConfiguration(t *testing.T) { if tt.featureGateEnabled { auth.EnableObjectLevelWorkloadIdentity() + t.Cleanup(auth.DisableObjectLevelWorkloadIdentity) } - auth.SetDefaultServiceAccount(tt.defaultServiceAccount) - auth.SetDefaultKubeConfigServiceAccount(tt.defaultKubeConfigServiceAccount) - auth.SetDefaultDecryptionServiceAccount(tt.defaultDecryptionServiceAccount) - - t.Cleanup(func() { - auth.SetDefaultServiceAccount("") - auth.SetDefaultKubeConfigServiceAccount("") - auth.SetDefaultDecryptionServiceAccount("") - auth.DisableObjectLevelWorkloadIdentity() - }) - - result := auth.InconsistentObjectLevelConfiguration() + result := auth.InconsistentObjectLevelConfiguration( + tt.defaultServiceAccount, + tt.defaultKubeConfigServiceAccount, + tt.defaultDecryptionServiceAccount, + ) g.Expect(result).To(Equal(tt.expectInconsistent)) }) } diff --git a/auth/gcp/implementation.go b/auth/gcp/implementation.go index c624ddba3..8860ebe60 100644 --- a/auth/gcp/implementation.go +++ b/auth/gcp/implementation.go @@ -23,12 +23,15 @@ import ( "golang.org/x/oauth2/google" "golang.org/x/oauth2/google/externalaccount" "google.golang.org/api/container/v1" + "google.golang.org/api/impersonate" + "google.golang.org/api/option" ) // Implementation provides the required methods of the GCP libraries. type Implementation interface { DefaultTokenSource(ctx context.Context, scope ...string) (oauth2.TokenSource, error) NewTokenSource(ctx context.Context, conf externalaccount.Config) (oauth2.TokenSource, error) + CredentialsTokenSource(ctx context.Context, conf impersonate.CredentialsConfig, opts ...option.ClientOption) (oauth2.TokenSource, error) GetCluster(ctx context.Context, cluster string, client *container.Service) (*container.Cluster, error) } @@ -42,6 +45,10 @@ func (implementation) NewTokenSource(ctx context.Context, conf externalaccount.C return externalaccount.NewTokenSource(ctx, conf) } +func (implementation) CredentialsTokenSource(ctx context.Context, conf impersonate.CredentialsConfig, opts ...option.ClientOption) (oauth2.TokenSource, error) { + return impersonate.CredentialsTokenSource(ctx, conf, opts...) +} + func (implementation) GetCluster(ctx context.Context, cluster string, client *container.Service) (*container.Cluster, error) { return client.Projects.Locations.Clusters.Get(cluster).Context(ctx).Do() } diff --git a/auth/gcp/implementation_test.go b/auth/gcp/implementation_test.go index e38151613..91e183958 100644 --- a/auth/gcp/implementation_test.go +++ b/auth/gcp/implementation_test.go @@ -29,19 +29,25 @@ import ( "golang.org/x/oauth2" "golang.org/x/oauth2/google/externalaccount" "google.golang.org/api/container/v1" + "google.golang.org/api/impersonate" + "google.golang.org/api/option" ) type mockImplementation struct { t *testing.T - expectGKEAPICall bool + expectGKEAPICall bool + expectImpersonationAPICall bool - argConfig externalaccount.Config - argProxyURL *url.URL - argCluster string + argConfig externalaccount.Config + argProxyURL *url.URL + argCluster string + argImpersonateTarget string - returnToken *oauth2.Token - returnCluster *container.Cluster + returnToken *oauth2.Token + returnCluster *container.Cluster + returnImpersonatedToken *oauth2.Token + returnImpersonationErr error } func (m *mockImplementation) DefaultTokenSource(ctx context.Context, scope ...string) (oauth2.TokenSource, error) { @@ -110,3 +116,20 @@ func (m *mockImplementation) GetCluster(ctx context.Context, cluster string, cli g.Expect(proxyURL).To(Equal(m.argProxyURL)) return m.returnCluster, nil } + +func (m *mockImplementation) CredentialsTokenSource(ctx context.Context, conf impersonate.CredentialsConfig, opts ...option.ClientOption) (oauth2.TokenSource, error) { + m.t.Helper() + g := NewWithT(m.t) + g.Expect(m.expectImpersonationAPICall).To(BeTrue()) + g.Expect(ctx).NotTo(BeNil()) + g.Expect(conf.TargetPrincipal).To(Equal(m.argImpersonateTarget)) + g.Expect(conf.Scopes).To(Equal([]string{ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/userinfo.email", + })) + g.Expect(opts).To(HaveLen(1)) + if m.returnImpersonationErr != nil { + return nil, m.returnImpersonationErr + } + return oauth2.StaticTokenSource(m.returnImpersonatedToken), nil +} diff --git a/auth/gcp/options.go b/auth/gcp/options.go index 1e386d948..2434418c5 100644 --- a/auth/gcp/options.go +++ b/auth/gcp/options.go @@ -17,10 +17,16 @@ limitations under the License. package gcp import ( + "context" "fmt" + "net/http" "regexp" + "google.golang.org/api/option" + htransport "google.golang.org/api/transport/http" corev1 "k8s.io/api/core/v1" + + "github.com/fluxcd/pkg/auth" ) const serviceAccountEmailPattern = `^[a-zA-Z0-9-]{1,100}@[a-zA-Z0-9-]{1,100}\.iam\.gserviceaccount\.com$` @@ -45,7 +51,7 @@ const workloadIdentityProviderPattern = `^projects/\d{1,30}/locations/global/wor var workloadIdentityProviderRegex = regexp.MustCompile(workloadIdentityProviderPattern) func getWorkloadIdentityProviderAudience(serviceAccount corev1.ServiceAccount) (string, error) { - const key = "gcp.auth.fluxcd.io/workload-identity-provider" + const key = ProviderName + "." + auth.APIGroup + "/workload-identity-provider" wip := serviceAccount.Annotations[key] if wip == "" { return "", nil @@ -68,3 +74,15 @@ func parseCluster(cluster string) error { } return nil } + +func newHTTPClient(ctx context.Context, token auth.Token, o *auth.Options) (*http.Client, error) { + baseTransport := http.DefaultTransport.(*http.Transport).Clone() + if p := o.ProxyURL; p != nil { + baseTransport.Proxy = http.ProxyURL(p) + } + transport, err := htransport.NewTransport(ctx, baseTransport, option.WithTokenSource(token.(*Token).source())) + if err != nil { + return nil, err + } + return &http.Client{Transport: transport}, nil +} diff --git a/auth/gcp/provider.go b/auth/gcp/provider.go index 71dc5bbc2..e06d57a36 100644 --- a/auth/gcp/provider.go +++ b/auth/gcp/provider.go @@ -19,16 +19,16 @@ package gcp import ( "context" "encoding/base64" + "encoding/json" "fmt" - "net/http" "regexp" "github.com/google/go-containerregistry/pkg/authn" "golang.org/x/oauth2" "golang.org/x/oauth2/google/externalaccount" "google.golang.org/api/container/v1" + "google.golang.org/api/impersonate" "google.golang.org/api/option" - htransport "google.golang.org/api/transport/http" corev1 "k8s.io/api/core/v1" auth "github.com/fluxcd/pkg/auth" @@ -157,6 +157,60 @@ func (p Provider) NewTokenForServiceAccount(ctx context.Context, oidcToken strin return &Token{*token}, nil } +// GetImpersonationAnnotationKey implements auth.ProviderWithImpersonation. +func (Provider) GetImpersonationAnnotationKey() string { + return "impersonate" +} + +type impersonation struct { + GCPServiceAccount string `json:"gcpServiceAccount"` +} + +func (i impersonation) String() string { + return i.GCPServiceAccount +} + +// GetIdentityForImpersonation implements auth.ProviderWithImpersonation. +func (Provider) GetIdentityForImpersonation(identity json.RawMessage) (fmt.Stringer, error) { + var id impersonation + if err := json.Unmarshal(identity, &id); err != nil { + return nil, fmt.Errorf("failed to unmarshal impersonation identity: %w", err) + } + if !serviceAccountEmailRegex.MatchString(id.GCPServiceAccount) { + return nil, fmt.Errorf("invalid GCP service account in impersonation identity: '%s'. must match %s", + id.GCPServiceAccount, serviceAccountEmailPattern) + } + return &id, nil +} + +// NewTokenForIdentity implements auth.ProviderWithImpersonation. +func (p Provider) NewTokenForIdentity(ctx context.Context, token auth.Token, + identity fmt.Stringer, opts ...auth.Option) (auth.Token, error) { + + var o auth.Options + o.Apply(opts...) + + hc, err := newHTTPClient(ctx, token, &o) + if err != nil { + return nil, fmt.Errorf("failed to create HTTP client for impersonation: %w", err) + } + + src, err := p.impl().CredentialsTokenSource(ctx, impersonate.CredentialsConfig{ + TargetPrincipal: identity.(*impersonation).GCPServiceAccount, + Scopes: scopes, + }, option.WithHTTPClient(hc)) + if err != nil { + return nil, fmt.Errorf("failed to create impersonated token source: %w", err) + } + + tok, err := src.Token() + if err != nil { + return nil, fmt.Errorf("failed to retrieve impersonated token: %w", err) + } + + return &Token{*tok}, nil +} + // GetAccessTokenOptionsForArtifactRepository implements auth.Provider. func (Provider) GetAccessTokenOptionsForArtifactRepository(string) ([]auth.Option, error) { // GCP does not require any special options to retrieve access tokens. @@ -224,15 +278,11 @@ func (p Provider) NewRESTConfig(ctx context.Context, accessTokens []auth.Token, } // Create client for describing the cluster resource. - baseTransport := http.DefaultTransport.(*http.Transport).Clone() - if p := o.ProxyURL; p != nil { - baseTransport.Proxy = http.ProxyURL(p) - } - transport, err := htransport.NewTransport(ctx, baseTransport, option.WithTokenSource(token.source())) + hc, err := newHTTPClient(ctx, token, &o) if err != nil { - return nil, fmt.Errorf("failed to create google http transport for describing GKE cluster: %w", err) + return nil, fmt.Errorf("failed to create HTTP client for describing GKE cluster: %w", err) } - client, err := container.NewService(ctx, option.WithHTTPClient(&http.Client{Transport: transport})) + client, err := container.NewService(ctx, option.WithHTTPClient(hc)) if err != nil { return nil, fmt.Errorf("failed to create client for describing GKE cluster: %w", err) } diff --git a/auth/gcp/provider_test.go b/auth/gcp/provider_test.go index 82f203ed5..e1307407d 100644 --- a/auth/gcp/provider_test.go +++ b/auth/gcp/provider_test.go @@ -18,6 +18,7 @@ package gcp_test import ( "context" + "errors" "net/url" "testing" "time" @@ -461,3 +462,125 @@ func TestProvider_GetAccessTokenOptionsForCluster(t *testing.T) { g.Expect(opts[0]).To(HaveLen(0)) // Empty slice - no options needed for GCP }) } + +func TestProvider_GetImpersonationAnnotationKey(t *testing.T) { + g := NewWithT(t) + g.Expect(gcp.Provider{}.GetImpersonationAnnotationKey()).To(Equal("impersonate")) +} + +func TestProvider_GetIdentityForImpersonation(t *testing.T) { + for _, tt := range []struct { + name string + identity string + expected string + err string + }{ + { + name: "valid GCP service account", + identity: `{"gcpServiceAccount":"test-sa@project-id.iam.gserviceaccount.com"}`, + expected: "test-sa@project-id.iam.gserviceaccount.com", + }, + { + name: "invalid GCP service account", + identity: `{"gcpServiceAccount":"foobar"}`, + err: "invalid GCP service account in impersonation identity: 'foobar'", + }, + { + name: "empty GCP service account", + identity: `{"gcpServiceAccount":""}`, + err: "invalid GCP service account in impersonation identity: ''", + }, + { + name: "invalid JSON", + identity: `{invalid`, + err: "failed to unmarshal impersonation identity", + }, + } { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + identity, err := gcp.Provider{}.GetIdentityForImpersonation([]byte(tt.identity)) + + if tt.err != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.err)) + g.Expect(identity).To(BeNil()) + } else { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(identity).NotTo(BeNil()) + g.Expect(identity.String()).To(Equal(tt.expected)) + } + }) + } +} + +func TestProvider_NewTokenForIdentity(t *testing.T) { + for _, tt := range []struct { + name string + gcpServiceAccount string + impersonationErr error + err string + }{ + { + name: "valid", + gcpServiceAccount: "target-sa@project-id.iam.gserviceaccount.com", + }, + { + name: "impersonation error", + gcpServiceAccount: "target-sa@project-id.iam.gserviceaccount.com", + impersonationErr: errors.New("impersonation failed"), + err: "failed to create impersonated token source: impersonation failed", + }, + } { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + tokenExpiry := time.Now().Add(1 * time.Hour) + impl := &mockImplementation{ + t: t, + expectImpersonationAPICall: true, + argProxyURL: &url.URL{Scheme: "http", Host: "proxy.example.com"}, + argImpersonateTarget: tt.gcpServiceAccount, + returnToken: &oauth2.Token{ + AccessToken: "initial-token", + Expiry: tokenExpiry, + }, + returnImpersonatedToken: &oauth2.Token{ + AccessToken: "impersonated-token", + Expiry: tokenExpiry, + }, + returnImpersonationErr: tt.impersonationErr, + } + + // Create the identity via GetIdentityForImpersonation. + identity, err := gcp.Provider{}.GetIdentityForImpersonation( + []byte(`{"gcpServiceAccount":"` + tt.gcpServiceAccount + `"}`)) + g.Expect(err).NotTo(HaveOccurred()) + + // Create a mock initial token. + initialToken := &gcp.Token{Token: oauth2.Token{ + AccessToken: "initial-token", + Expiry: tokenExpiry, + }} + + opts := []auth.Option{ + auth.WithProxyURL(url.URL{Scheme: "http", Host: "proxy.example.com"}), + } + + provider := gcp.Provider{Implementation: impl} + token, err := provider.NewTokenForIdentity(context.Background(), initialToken, identity, opts...) + + if tt.err != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.err)) + g.Expect(token).To(BeNil()) + } else { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(token).To(Equal(&gcp.Token{Token: oauth2.Token{ + AccessToken: "impersonated-token", + Expiry: tokenExpiry, + }})) + } + }) + } +} diff --git a/auth/go.mod b/auth/go.mod index 34c525b64..a5bb7c687 100644 --- a/auth/go.mod +++ b/auth/go.mod @@ -33,6 +33,7 @@ require ( k8s.io/apimachinery v0.35.0 k8s.io/client-go v0.35.0 sigs.k8s.io/controller-runtime v0.23.0 + sigs.k8s.io/yaml v1.6.0 ) require ( @@ -114,5 +115,4 @@ require ( sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect - sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/auth/go.sum b/auth/go.sum index 02fd05604..a02518d6f 100644 --- a/auth/go.sum +++ b/auth/go.sum @@ -190,6 +190,8 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 h1:q4XOmH/0opmeuJtPsbFNivyl7bCt7yRBbeEm2sC/XtQ= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg= go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= diff --git a/auth/options.go b/auth/options.go index afcf3ba82..4c0474c2c 100644 --- a/auth/options.go +++ b/auth/options.go @@ -36,6 +36,7 @@ type Options struct { Cache *cache.TokenCache ServiceAccountName string ServiceAccountNamespace string + DefaultServiceAccount string InvolvedObject cache.InvolvedObject Audiences []string Scopes []string @@ -48,13 +49,9 @@ type Options struct { AllowShellOut bool } -// ShouldGetServiceAccountToken returns true if ServiceAccount token should be retrieved. -func (o *Options) ShouldGetServiceAccountToken() bool { - // ServiceAccount namespace is required because ServiceAccounts are namespace-scoped resources. - // ServiceAccountName can be empty as it may be provided by defaultServiceAccount or by - // defaultKubeConfigServiceAccount. - return o.ServiceAccountNamespace != "" && - (o.ServiceAccountName != "" || getDefaultServiceAccount() != "") +// ShouldGetServiceAccount returns true if a ServiceAccount should be retrieved. +func (o *Options) ShouldGetServiceAccount() bool { + return o.ServiceAccountName != "" || o.DefaultServiceAccount != "" } // WithClient sets the controller-runtime client for the provider. @@ -78,6 +75,14 @@ func WithServiceAccountNamespace(namespace string) Option { } } +// WithDefaultServiceAccount sets the default ServiceAccount name for the token +// if ServiceAccountName is not provided. +func WithDefaultServiceAccount(name string) Option { + return func(o *Options) { + o.DefaultServiceAccount = name + } +} + // WithCache sets the token cache and the involved object for recording events. func WithCache(cache cache.TokenCache, involvedObject cache.InvolvedObject) Option { return func(o *Options) { diff --git a/auth/options_test.go b/auth/options_test.go index cb173eee3..4a28239b8 100644 --- a/auth/options_test.go +++ b/auth/options_test.go @@ -70,91 +70,56 @@ func TestOptions_GetHTTPClient(t *testing.T) { } } -func TestOptions_ShouldGetServiceAccountToken(t *testing.T) { +func TestOptions_ShouldGetServiceAccount(t *testing.T) { tests := []struct { - name string - opts []auth.Option - defaultSA string - defaultKubeConfigSA string - defaultDecryptionSA string - expected bool + name string + opts []auth.Option + expected bool }{ { - name: "both name and namespace provided", + name: "service account name provided", opts: []auth.Option{ auth.WithServiceAccountName("test-sa"), - auth.WithServiceAccountNamespace("default"), }, expected: true, }, { - name: "only namespace provided - no global vars", + name: "default service account provided", opts: []auth.Option{ - auth.WithServiceAccountNamespace("default"), + auth.WithDefaultServiceAccount("default-sa"), }, - expected: false, + expected: true, }, { - name: "namespace and defaultServiceAccount", + name: "both name and default provided", opts: []auth.Option{ - auth.WithServiceAccountNamespace("default"), + auth.WithServiceAccountName("test-sa"), + auth.WithDefaultServiceAccount("default-sa"), }, - defaultSA: "default-sa", - expected: true, + expected: true, }, { - name: "namespace and defaultKubeConfigServiceAccount", - opts: []auth.Option{ - auth.WithServiceAccountNamespace("default"), - }, - defaultKubeConfigSA: "default-kubeconfig-sa", - expected: true, + name: "neither provided", + opts: []auth.Option{}, + expected: false, }, { - name: "namespace and defaultDecryptionServiceAccount - expect false! decryption is handled in kustomize-controller", + name: "only namespace provided", opts: []auth.Option{ auth.WithServiceAccountNamespace("default"), }, - defaultDecryptionSA: "default-decryption-sa", - expected: false, - }, - { - name: "only name provided", - opts: []auth.Option{ - auth.WithServiceAccountName("test-sa"), - }, - expected: false, - }, - { - name: "neither provided", - opts: []auth.Option{}, expected: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if tt.defaultSA != "" { - auth.SetDefaultServiceAccount(tt.defaultSA) - t.Cleanup(func() { auth.SetDefaultServiceAccount("") }) - } - - if tt.defaultKubeConfigSA != "" { - auth.SetDefaultKubeConfigServiceAccount(tt.defaultKubeConfigSA) - t.Cleanup(func() { auth.SetDefaultKubeConfigServiceAccount("") }) - } - - if tt.defaultDecryptionSA != "" { - auth.SetDefaultDecryptionServiceAccount(tt.defaultDecryptionSA) - t.Cleanup(func() { auth.SetDefaultDecryptionServiceAccount("") }) - } - var o auth.Options o.Apply(tt.opts...) - result := o.ShouldGetServiceAccountToken() + result := o.ShouldGetServiceAccount() if result != tt.expected { - t.Errorf("ShouldGetServiceAccountToken() = %v, want %v", result, tt.expected) + t.Errorf("ShouldGetServiceAccount() = %v, want %v", result, tt.expected) } }) } diff --git a/auth/provider.go b/auth/provider.go index c780d91db..0ee5fe33b 100644 --- a/auth/provider.go +++ b/auth/provider.go @@ -18,6 +18,8 @@ package auth import ( "context" + "encoding/json" + "fmt" corev1 "k8s.io/api/core/v1" ) @@ -52,3 +54,30 @@ type Provider interface { NewTokenForServiceAccount(ctx context.Context, oidcToken string, serviceAccount corev1.ServiceAccount, opts ...Option) (Token, error) } + +// ProviderWithImpersonation is an optional interface that providers can +// implement if they support impersonation of identities. For example, AWS +// IAM identities can impersonate AWS IAM Roles with the AssumeRole API. +// Note that the impersonation type here is cloud identity -> cloud identity, +// and not Kubernetes ServiceAccount -> cloud identity, which is what +// NewTokenForServiceAccount is for. +type ProviderWithImpersonation interface { + Provider + + // GetImpersonationAnnotationKey returns the annotation key without API group + // that should be used in the ServiceAccount to specify the provider identity + // to impersonate. + GetImpersonationAnnotationKey() string + + // GetIdentityForImpersonation takes the marshaled impersonation + // configuration from the ServiceAccount annotations and returns + // the identity that should be impersonated with the initially + // acquired cloud provider access token. + GetIdentityForImpersonation(identity json.RawMessage) (fmt.Stringer, error) + + // NewTokenForIdentity takes a provider token and identity and + // returns another provider token that can be used to authenticate + // with the cloud provider impersonating the given identity. + NewTokenForIdentity(ctx context.Context, token Token, + identity fmt.Stringer, opts ...Option) (Token, error) +} diff --git a/auth/provider_test.go b/auth/provider_test.go index cdf07a2a0..ea2d97cdd 100644 --- a/auth/provider_test.go +++ b/auth/provider_test.go @@ -18,7 +18,9 @@ package auth_test import ( "context" + "encoding/json" "errors" + "fmt" "net/http" "net/url" "testing" @@ -31,6 +33,11 @@ import ( "github.com/fluxcd/pkg/auth" ) +// mockIdentity implements fmt.Stringer for testing impersonation identities. +type mockIdentity string + +func (m mockIdentity) String() string { return string(m) } + type mockProvider struct { t *testing.T @@ -60,6 +67,43 @@ type mockProvider struct { paramSecondScopes []string expectFirstScopes bool expectSecondScopes bool + + // Impersonation fields (used when wrapped with mockProviderWithImpersonation). + returnImpersonationAnnotationKey string + returnIdentityForImpersonation fmt.Stringer + returnIdentityForImpersonationErr string + returnImpersonatedToken auth.Token + returnImpersonateErr error + gotIdentity fmt.Stringer +} + +// mockProviderWithImpersonation wraps mockProvider to also implement +// auth.ProviderWithImpersonation. +type mockProviderWithImpersonation struct { + *mockProvider +} + +func (m *mockProviderWithImpersonation) GetImpersonationAnnotationKey() string { + if m.returnImpersonationAnnotationKey != "" { + return m.returnImpersonationAnnotationKey + } + return "impersonation" +} + +func (m *mockProviderWithImpersonation) GetIdentityForImpersonation(identity json.RawMessage) (fmt.Stringer, error) { + if m.returnIdentityForImpersonationErr != "" { + return nil, errors.New(m.returnIdentityForImpersonationErr) + } + return m.returnIdentityForImpersonation, nil +} + +func (m *mockProviderWithImpersonation) NewTokenForIdentity(ctx context.Context, token auth.Token, + identity fmt.Stringer, opts ...auth.Option) (auth.Token, error) { + m.gotIdentity = identity + if m.returnImpersonateErr != nil { + return nil, m.returnImpersonateErr + } + return m.returnImpersonatedToken, nil } func (m *mockProvider) GetName() string { diff --git a/auth/registry.go b/auth/registry.go index babad8819..d7446c3c8 100644 --- a/auth/registry.go +++ b/auth/registry.go @@ -24,8 +24,6 @@ import ( "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" - corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" "github.com/fluxcd/pkg/cache" ) @@ -119,23 +117,15 @@ func GetArtifactRegistryCredentials(ctx context.Context, provider ArtifactRegist } // Build cache key. - var serviceAccount *corev1.ServiceAccount - var providerIdentity string - var audiences []string - if o.ShouldGetServiceAccountToken() { + var saInfo *serviceAccountInfo + if o.ShouldGetServiceAccount() { var err error - saRef := client.ObjectKey{ - Name: o.ServiceAccountName, - Namespace: o.ServiceAccountNamespace, - } - serviceAccount, audiences, providerIdentity, err = - getServiceAccountAndProviderInfo(ctx, provider, o.Client, saRef, opts...) + saInfo, err = getServiceAccountInfo(ctx, provider, o.Client, opts...) if err != nil { return nil, err } } - accessTokenCacheKey := buildAccessTokenCacheKey(provider, audiences, - providerIdentity, serviceAccount, accessTokenOpts...) + accessTokenCacheKey := buildAccessTokenCacheKey(provider, saInfo, accessTokenOpts...) cacheKey := buildCacheKey( fmt.Sprintf("accessTokenCacheKey=%s", accessTokenCacheKey), fmt.Sprintf("artifactRepositoryCacheKey=%s", registryInput)) diff --git a/auth/registry_test.go b/auth/registry_test.go index 17675f657..1f8d2854a 100644 --- a/auth/registry_test.go +++ b/auth/registry_test.go @@ -125,7 +125,6 @@ func TestGetArtifactRegistryCredentials(t *testing.T) { artifactRepository string opts []auth.Option disableObjectLevel bool - defaultSA string expectedCreds *auth.ArtifactRegistryCredentials expectedErr string }{ @@ -209,8 +208,8 @@ func TestGetArtifactRegistryCredentials(t *testing.T) { auth.WithSTSEndpoint("https://sts.some-cloud.io"), auth.WithProxyURL(url.URL{Scheme: "http", Host: "proxy.io:8080"}), auth.WithCAData("ca-data"), + auth.WithDefaultServiceAccount(saRef.Name), }, - defaultSA: saRef.Name, expectedCreds: &auth.ArtifactRegistryCredentials{ Authenticator: authn.FromConfig(authn.AuthConfig{Username: "mock-registry-token"}), }, @@ -240,8 +239,8 @@ func TestGetArtifactRegistryCredentials(t *testing.T) { auth.WithSTSEndpoint("https://sts.some-cloud.io"), auth.WithProxyURL(url.URL{Scheme: "http", Host: "proxy.io:8080"}), auth.WithCAData("ca-data"), + auth.WithDefaultServiceAccount(saRef.Name), }, - defaultSA: saRef.Name, disableObjectLevel: true, expectedCreds: &auth.ArtifactRegistryCredentials{ Authenticator: authn.FromConfig(authn.AuthConfig{Username: "mock-registry-token"}), @@ -272,7 +271,7 @@ func TestGetArtifactRegistryCredentials(t *testing.T) { tokenCache, err := cache.NewTokenCache(2) g.Expect(err).NotTo(HaveOccurred()) - const accessTokenKey = "db625bd5a96dc48fcc100659c6db98857d1e0ceec930bbded0fdece14af4307c" + const accessTokenKey = "6fbdfd364d87e47e6aad554232b927805c949ac461c43eb1c84d7dbcd58c38fb" var token auth.Token = &mockToken{token: "cached-token"} cachedToken, ok, err := tokenCache.GetOrSet(ctx, accessTokenKey, func(ctx context.Context) (cache.Token, error) { return token, nil @@ -281,7 +280,7 @@ func TestGetArtifactRegistryCredentials(t *testing.T) { g.Expect(ok).To(BeFalse()) g.Expect(cachedToken).To(Equal(token)) - const artifactRegistryCredentialsKey = "61fe71ebbf306060d67acbdc2389d5fd816bee40e7685afe2fdc18b7d3bde1d6" + const artifactRegistryCredentialsKey = "3b08cea6e7afd6072c6e52b6b3e19e5896668d2a5301ff68cc403dc611554eef" token = &auth.ArtifactRegistryCredentials{ Authenticator: authn.FromConfig(authn.AuthConfig{Username: "cached-registry-token"}), ExpiresAt: now.Add(time.Hour), @@ -340,11 +339,6 @@ func TestGetArtifactRegistryCredentials(t *testing.T) { t.Cleanup(auth.DisableObjectLevelWorkloadIdentity) } - if tt.defaultSA != "" { - auth.SetDefaultServiceAccount(tt.defaultSA) - t.Cleanup(func() { auth.SetDefaultServiceAccount("") }) - } - creds, err := auth.GetArtifactRegistryCredentials(ctx, tt.provider, tt.artifactRepository, tt.opts...) if tt.expectedErr != "" { diff --git a/auth/restconfig.go b/auth/restconfig.go index 26dc38feb..04a23ec23 100644 --- a/auth/restconfig.go +++ b/auth/restconfig.go @@ -25,9 +25,6 @@ import ( "strings" "time" - corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/fluxcd/pkg/cache" ) @@ -128,24 +125,17 @@ func GetRESTConfig(ctx context.Context, provider RESTConfigProvider, opts ...Opt } // Build cache key. - var serviceAccount *corev1.ServiceAccount - var providerIdentity string - var audiences []string - if o.ShouldGetServiceAccountToken() { + var saInfo *serviceAccountInfo + if o.ShouldGetServiceAccount() { var err error - saRef := client.ObjectKey{ - Name: o.ServiceAccountName, - Namespace: o.ServiceAccountNamespace, - } - serviceAccount, audiences, providerIdentity, err = - getServiceAccountAndProviderInfo(ctx, provider, o.Client, saRef, opts...) + saInfo, err = getServiceAccountInfo(ctx, provider, o.Client, opts...) if err != nil { return nil, err } } var cacheKeyParts []string for i, atOpts := range accessTokenOpts { - key := buildAccessTokenCacheKey(provider, audiences, providerIdentity, serviceAccount, atOpts...) + key := buildAccessTokenCacheKey(provider, saInfo, atOpts...) cacheKeyParts = append(cacheKeyParts, fmt.Sprintf("accessToken%dCacheKey=%s", i, key)) } if c := o.ClusterResource; c != "" { diff --git a/auth/restconfig_test.go b/auth/restconfig_test.go index 153a70f24..0c79ab454 100644 --- a/auth/restconfig_test.go +++ b/auth/restconfig_test.go @@ -121,14 +121,13 @@ func TestGetRESTConfig(t *testing.T) { now := time.Now() for _, tt := range []struct { - name string - provider *mockProvider - cluster string - opts []auth.Option - disableObjectLevel bool - defaultKubeConfigSA string - expectedCreds *auth.RESTConfig - expectedErr string + name string + provider *mockProvider + cluster string + opts []auth.Option + disableObjectLevel bool + expectedCreds *auth.RESTConfig + expectedErr string }{ { name: "restconfig from controller access token", @@ -236,7 +235,7 @@ func TestGetRESTConfig(t *testing.T) { tokenCache, err := cache.NewTokenCache(3) g.Expect(err).NotTo(HaveOccurred()) - accessTokenKey := "500a3116f5d1c492d7a5ea97cdf9a7f869815346c79f01c7368703c241ebb5eb" + accessTokenKey := "3db1f35173795b209e61309a55f292c1696023a5fa5138ad9b32ba38f02720d4" var token auth.Token = &mockToken{token: "cached-token"} cachedToken, ok, err := tokenCache.GetOrSet(ctx, accessTokenKey, func(ctx context.Context) (cache.Token, error) { return token, nil @@ -245,7 +244,7 @@ func TestGetRESTConfig(t *testing.T) { g.Expect(ok).To(BeFalse()) g.Expect(cachedToken).To(Equal(token)) - accessTokenKey = "0b1167fc851943c6153d40e149cd2970aac121aaf03b1fcad158672974f58827" + accessTokenKey = "92cb5ac8dd7147e4d9f57d4675497ef6ccf0fafcdbcc6d268901964371f8040c" token = &mockToken{token: "cached-token"} cachedToken, ok, err = tokenCache.GetOrSet(ctx, accessTokenKey, func(ctx context.Context) (cache.Token, error) { return token, nil @@ -254,7 +253,7 @@ func TestGetRESTConfig(t *testing.T) { g.Expect(ok).To(BeFalse()) g.Expect(cachedToken).To(Equal(token)) - const restConfigKey = "a1937b7b1df13ac8ad784db686088c4cd5b4c4877318d07d3fa19ab8caf9d7c2" + const restConfigKey = "5666bcc9d6bed56d22e203ac7956db030464fc1b8f3e677ac32297b2db60b240" token = &auth.RESTConfig{ Host: "https://cluster/resource/name", BearerToken: "mock-bearer-token", @@ -320,8 +319,8 @@ func TestGetRESTConfig(t *testing.T) { auth.WithSTSEndpoint("https://sts.some-cloud.io"), auth.WithProxyURL(url.URL{Scheme: "http", Host: "proxy.io:8080"}), auth.WithCAData("ca-data"), + auth.WithDefaultServiceAccount("lockdown-sa"), }, - defaultKubeConfigSA: "lockdown-sa", expectedCreds: &auth.RESTConfig{ Host: "https://cluster/resource/name", BearerToken: "mock-bearer-token", @@ -361,9 +360,9 @@ func TestGetRESTConfig(t *testing.T) { auth.WithSTSEndpoint("https://sts.some-cloud.io"), auth.WithProxyURL(url.URL{Scheme: "http", Host: "proxy.io:8080"}), auth.WithCAData("ca-data"), + auth.WithDefaultServiceAccount("lockdown-sa"), }, - defaultKubeConfigSA: "lockdown-sa", - disableObjectLevel: true, + disableObjectLevel: true, expectedCreds: &auth.RESTConfig{ Host: "https://cluster/resource/name", BearerToken: "mock-bearer-token", @@ -405,11 +404,6 @@ func TestGetRESTConfig(t *testing.T) { t.Cleanup(auth.DisableObjectLevelWorkloadIdentity) } - if tt.defaultKubeConfigSA != "" { - auth.SetDefaultKubeConfigServiceAccount(tt.defaultKubeConfigSA) - t.Cleanup(func() { auth.SetDefaultKubeConfigServiceAccount("") }) - } - if tt.cluster != "" { tt.opts = append(tt.opts, auth.WithClusterResource(tt.cluster)) } diff --git a/auth/service_account.go b/auth/service_account.go new file mode 100644 index 000000000..94cfeeb8a --- /dev/null +++ b/auth/service_account.go @@ -0,0 +1,158 @@ +/* +Copyright 2026 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" +) + +// serviceAccountInfo contains the parsed information of the ServiceAccount +// to be used for fetching the access token and generating the cache key when +// object-level workload identity is enabled. +type serviceAccountInfo struct { + useServiceAccount bool + obj *corev1.ServiceAccount + audiences []string + providerIdentity string + providerIdentityForImpersonation fmt.Stringer +} + +// getServiceAccountInfo fetches the ServiceAccount and parses the necessary information for +// fetching the access token and generating the cache key when object-level workload identity +// is enabled. +func getServiceAccountInfo(ctx context.Context, provider Provider, + client client.Client, opts ...Option) (*serviceAccountInfo, error) { + + var o Options + o.Apply(opts...) + + key := types.NamespacedName{ + Name: o.ServiceAccountName, + Namespace: o.ServiceAccountNamespace, + } + + // Apply multi-tenancy lockdown: use default service account when .serviceAccountName + // is not explicitly specified in the object. This results in Object-Level Workload Identity. + var setDefaultSA bool + lockdownEnabled := o.DefaultServiceAccount != "" + if key.Name == "" && lockdownEnabled { + key.Name = o.DefaultServiceAccount + setDefaultSA = true + } + + // Get service account. + var obj corev1.ServiceAccount + if err := client.Get(ctx, key, &obj); err != nil { + if errors.IsNotFound(err) && setDefaultSA { + return nil, fmt.Errorf("failed to get service account '%s': %w", + key, ErrDefaultServiceAccountNotFound) + } + return nil, fmt.Errorf("failed to get service account '%s': %w", key, err) + } + + // If no impersonation configuration is found, default to using the ServiceAccount with the provider. + useServiceAccount := true + + // Get provider identity for impersonation if supported by the provider. + var providerIdentityForImpersonation fmt.Stringer + if provider, ok := provider.(ProviderWithImpersonation); ok { + annotationKey := ImpersonationAnnotation(provider) + if impersonationYAML := obj.Annotations[annotationKey]; impersonationYAML != "" { + // Parse the impersonation configuration from the annotation. + var impersonation Impersonation + if err := yaml.Unmarshal([]byte(impersonationYAML), &impersonation); err != nil { + return nil, fmt.Errorf( + "failed to parse impersonation annotation '%s' on service account '%s': %w", + annotationKey, key, err) + } + var err error + providerIdentityForImpersonation, err = provider.GetIdentityForImpersonation(impersonation.Identity) + if err != nil { + return nil, fmt.Errorf( + "failed to get provider identity for impersonation from service account '%s' annotation '%s': %w", + key, annotationKey, err) + } + + // If UseServiceAccount is not set, default to true if lockdown is enabled, and false otherwise. + if impersonation.UseServiceAccount == nil { + impersonation.UseServiceAccount = &lockdownEnabled + } + useServiceAccount = *impersonation.UseServiceAccount + + // If the user intention is to not use the ServiceAccount, but the cluster + // administrator has enabled multi-tenancy lockdown, return an error. + if !useServiceAccount && lockdownEnabled { + return nil, fmt.Errorf("invalid impersonation configuration on service account '%s': "+ + "multi-tenancy lockdown is enabled, impersonation without service account is not allowed", key) + } + } + } + + var audiences []string + var providerIdentity string + + switch useServiceAccount { + + // If the user intention is to use the ServiceAccount, + // get the required fields for the usage. + case true: + // Get provider audience. + audiences = o.Audiences + if len(audiences) == 0 { + var err error + audiences, err = provider.GetAudiences(ctx, obj) + if err != nil { + return nil, fmt.Errorf("failed to get provider audience: %w", err) + } + } + + // Get provider identity. + var err error + providerIdentity, err = provider.GetIdentity(obj) + if err != nil { + return nil, fmt.Errorf( + "failed to get provider identity from service account '%s' annotations: %w", key, err) + } + + // If the user does not want to use the ServiceAccount, error out if + // unneeded annotations are present to avoid confusion. + case false: + // No need to check audiences, they may be needed for usage without a ServiceAccount. + + // Check provider identity. + if id, err := provider.GetIdentity(obj); err == nil && id != "" { + return nil, fmt.Errorf("invalid configuration on service account '%s': "+ + "identity annotation is present but the ServiceAccount is not used according "+ + "to the impersonation configuration", key) + } + } + + return &serviceAccountInfo{ + useServiceAccount: useServiceAccount, + obj: &obj, + audiences: audiences, + providerIdentity: providerIdentity, + providerIdentityForImpersonation: providerIdentityForImpersonation, + }, nil +} diff --git a/tests/integration/aws_test.go b/tests/integration/aws_test.go index 14df772fc..5b0b1a0f0 100644 --- a/tests/integration/aws_test.go +++ b/tests/integration/aws_test.go @@ -129,13 +129,65 @@ func getClusterConfigMapAWS(output map[string]*tfjson.StateOutput) (map[string]s }, nil } +// getImpersonationAnnotationsAWS returns annotations for impersonation target SAs. +func getImpersonationAnnotationsAWS(output map[string]*tfjson.StateOutput) (map[string]map[string]string, error) { + targetARN := output["aws_assume_role_target_arn"].Value.(string) + if targetARN == "" { + return nil, fmt.Errorf("no target role ARN in terraform output") + } + sourceARN := output["aws_wi_iam_arn"].Value.(string) + if sourceARN == "" { + return nil, fmt.Errorf("no source role ARN in terraform output") + } + + return map[string]map[string]string{ + // Controller-level target SA (useServiceAccount: false). + wiAssumeRoleCtrlSA: { + "aws.auth.fluxcd.io/assume-role": fmt.Sprintf( + `{"roleARN":"%s","useServiceAccount":false}`, targetARN), + }, + // Object-level target SA with IRSA (useServiceAccount: true). + wiAssumeRoleSA: { + eksRoleArnAnnotation: sourceARN, + "aws.auth.fluxcd.io/assume-role": fmt.Sprintf( + `{"roleARN":"%s","useServiceAccount":true}`, targetARN), + }, + }, nil +} + +// getControllerAnnotationsAWS returns annotations for controller SAs used in +// impersonation testing. +func getControllerAnnotationsAWS(output map[string]*tfjson.StateOutput) (map[string]map[string]string, error) { + controllerIRSAArn := output["aws_controller_irsa_arn"].Value.(string) + if controllerIRSAArn == "" { + return nil, fmt.Errorf("no controller IRSA role ARN in terraform output") + } + + return map[string]map[string]string{ + // Controller with IRSA credentials. + wiControllerIRSA: { + eksRoleArnAnnotation: controllerIRSAArn, + }, + // Controller with Pod Identity credentials (no annotations needed, + // Pod Identity Agent injects credentials). + wiControllerPodIdentity: {}, + }, nil +} + // getClusterUsersAWS returns the cluster users for kubeconfig auth tests. func getClusterUsersAWS(output map[string]*tfjson.StateOutput) ([]string, error) { clusterUser := output["aws_wi_iam_arn"].Value.(string) if clusterUser == "" { return nil, fmt.Errorf("no EKS cluster user id in terraform output") } - return []string{clusterUser}, nil + users := []string{clusterUser} + + // Include the assume role target ARN as a cluster user for restconfig tests. + if targetARN := output["aws_assume_role_target_arn"].Value.(string); targetARN != "" { + users = append(users, targetARN) + } + + return users, nil } // When implemented, getGitTestConfigAws would return the git-specific test config for AWS diff --git a/tests/integration/gcp_test.go b/tests/integration/gcp_test.go index cd52156e9..c07b0ded5 100644 --- a/tests/integration/gcp_test.go +++ b/tests/integration/gcp_test.go @@ -128,15 +128,70 @@ func getClusterUsersGCP(output map[string]*tfjson.StateOutput) ([]string, error) "wi_iam_serviceaccount_email", "wi_k8s_sa_principal_direct_access", "wi_k8s_sa_principal_direct_access_federation", + "impersonation_target_email", } { - if clusterUser := output[key].Value.(string); clusterUser != "" { - clusterUsers = append(clusterUsers, clusterUser) + if v, ok := output[key]; ok { + if clusterUser := v.Value.(string); clusterUser != "" { + clusterUsers = append(clusterUsers, clusterUser) + } } } return clusterUsers, nil } +// getImpersonationAnnotationsGCP returns annotations for impersonation target SAs. +func getImpersonationAnnotationsGCP(output map[string]*tfjson.StateOutput) (map[string]map[string]string, error) { + targetEmail := output["impersonation_target_email"].Value.(string) + if targetEmail == "" { + return nil, fmt.Errorf("no impersonation target email in terraform output") + } + saEmail := output["wi_iam_serviceaccount_email"].Value.(string) + if saEmail == "" { + return nil, fmt.Errorf("no GCP serviceaccount email in terraform output") + } + wip := output["workload_identity_provider"].Value.(string) + if wip == "" { + return nil, fmt.Errorf("no workload identity provider in terraform output") + } + + return map[string]map[string]string{ + // Controller-level target SA (useServiceAccount: false). + wiImpersonateCtrlSA: { + "gcp.auth.fluxcd.io/impersonate": fmt.Sprintf( + `{"gcpServiceAccount":"%s","useServiceAccount":false}`, targetEmail), + }, + // Object-level target SA with GCP SA (useServiceAccount: true). + wiImpersonateSA: { + gcpIAMAnnotation: saEmail, + "gcp.auth.fluxcd.io/impersonate": fmt.Sprintf( + `{"gcpServiceAccount":"%s","useServiceAccount":true}`, targetEmail), + }, + // Object-level target SA with WIF federation direct access (useServiceAccount: true). + wiImpersonateDirectAccessSA: { + gcpWorkloadIdentityProviderAnnotation: wip, + "gcp.auth.fluxcd.io/impersonate": fmt.Sprintf( + `{"gcpServiceAccount":"%s","useServiceAccount":true}`, targetEmail), + }, + }, nil +} + +// getControllerAnnotationsGCP returns annotations for controller SAs used in +// impersonation testing. +func getControllerAnnotationsGCP(output map[string]*tfjson.StateOutput) (map[string]map[string]string, error) { + saEmail := output["wi_iam_serviceaccount_email"].Value.(string) + if saEmail == "" { + return nil, fmt.Errorf("no GCP serviceaccount email in terraform output") + } + + return map[string]map[string]string{ + // Controller with GCP SA annotation (gets GCP SA-impersonated token from GKE metadata). + wiControllerGCPSA: { + gcpIAMAnnotation: saEmail, + }, + }, nil +} + // When implemented, getGitTestConfigGCP would return the git-specific test config for GCP func getGitTestConfigGCP(outputs map[string]*tfjson.StateOutput) (*gitTestConfig, error) { return nil, fmt.Errorf("NotImplemented for GCP") diff --git a/tests/integration/go.sum b/tests/integration/go.sum index f4491a886..f7342399b 100644 --- a/tests/integration/go.sum +++ b/tests/integration/go.sum @@ -323,6 +323,8 @@ github.com/zclconf/go-cty v1.16.4 h1:QGXaag7/7dCzb+odlGrgr+YmYZFaOCMW6DEpS+UD1eE github.com/zclconf/go-cty v1.16.4/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 h1:q4XOmH/0opmeuJtPsbFNivyl7bCt7yRBbeEm2sC/XtQ= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg= go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= diff --git a/tests/integration/job_test.go b/tests/integration/job_test.go index 98419db5f..3c830667d 100644 --- a/tests/integration/job_test.go +++ b/tests/integration/job_test.go @@ -38,6 +38,17 @@ const ( objectLevelWIModeImpersonation objectLevelWIModeDirectAccessFederation objectLevelWIModeImpersonationFederation + + // AWS AssumeRole impersonation modes. + objectLevelWIModeAssumeRoleIRSA // pod=flux-controller-irsa, wisa=test-workload-id-assume-role-ctrl + objectLevelWIModeAssumeRolePodIdentity // pod=flux-controller-pod-identity, wisa=test-workload-id-assume-role-ctrl + objectLevelWIModeAssumeRoleObjectLevel // pod=flux-controller, wisa=test-workload-id-assume-role + + // GCP impersonation modes. + objectLevelWIModeGCPImpersonateCtrl // pod=flux-controller, wisa=test-workload-id-impersonate-ctrl + objectLevelWIModeGCPImpersonateCtrlSA // pod=flux-controller-gcp-sa, wisa=test-workload-id-impersonate-ctrl + objectLevelWIModeGCPImpersonateObj // pod=flux-controller, wisa=test-workload-id-impersonate + objectLevelWIModeGCPImpersonateObjDA // pod=flux-controller, wisa=test-workload-id-impersonate-da ) type objectLevelWIMode int @@ -90,6 +101,23 @@ func testjobExecutionWithArgs(t *testing.T, args []string, opts ...jobOption) { args = append(args, "-wisa-name="+wiServiceAccountFederation) case objectLevelWIModeDirectAccessFederation: args = append(args, "-wisa-name="+wiServiceAccountFederationDirectAccess) + case objectLevelWIModeAssumeRoleIRSA: + saName = wiControllerIRSA + args = append(args, "-wisa-name="+wiAssumeRoleCtrlSA) + case objectLevelWIModeAssumeRolePodIdentity: + saName = wiControllerPodIdentity + args = append(args, "-wisa-name="+wiAssumeRoleCtrlSA) + case objectLevelWIModeAssumeRoleObjectLevel: + args = append(args, "-wisa-name="+wiAssumeRoleSA) + case objectLevelWIModeGCPImpersonateCtrl: + args = append(args, "-wisa-name="+wiImpersonateCtrlSA) + case objectLevelWIModeGCPImpersonateCtrlSA: + saName = wiControllerGCPSA + args = append(args, "-wisa-name="+wiImpersonateCtrlSA) + case objectLevelWIModeGCPImpersonateObj: + args = append(args, "-wisa-name="+wiImpersonateSA) + case objectLevelWIModeGCPImpersonateObjDA: + args = append(args, "-wisa-name="+wiImpersonateDirectAccessSA) } } job.Spec.Template.Spec.ServiceAccountName = saName diff --git a/tests/integration/oci_test.go b/tests/integration/oci_test.go index 902175fc1..e2ca2cb18 100644 --- a/tests/integration/oci_test.go +++ b/tests/integration/oci_test.go @@ -63,6 +63,41 @@ func TestOciImageRepositoryListTags(t *testing.T) { skip: !testWIDirectAccess || !testWIFederation, opts: []jobOption{withObjectLevelWI(objectLevelWIModeDirectAccessFederation)}, }, + { + name: "impersonation: controller IRSA -> AssumeRole (AWS)", + skip: !testImpersonation || *targetProvider != "aws", + opts: []jobOption{withObjectLevelWI(objectLevelWIModeAssumeRoleIRSA)}, + }, + { + name: "impersonation: controller Pod Identity -> AssumeRole (AWS)", + skip: !testImpersonation || *targetProvider != "aws", + opts: []jobOption{withObjectLevelWI(objectLevelWIModeAssumeRolePodIdentity)}, + }, + { + name: "impersonation: object-level IRSA -> AssumeRole (AWS)", + skip: !testImpersonation || *targetProvider != "aws", + opts: []jobOption{withObjectLevelWI(objectLevelWIModeAssumeRoleObjectLevel)}, + }, + { + name: "impersonation: controller WIF -> Impersonate (GCP)", + skip: !testImpersonation || *targetProvider != "gcp", + opts: []jobOption{withObjectLevelWI(objectLevelWIModeGCPImpersonateCtrl)}, + }, + { + name: "impersonation: controller WIF+SA -> Impersonate (GCP)", + skip: !testImpersonation || *targetProvider != "gcp", + opts: []jobOption{withObjectLevelWI(objectLevelWIModeGCPImpersonateCtrlSA)}, + }, + { + name: "impersonation: object-level WIF+SA -> Impersonate (GCP)", + skip: !testImpersonation || *targetProvider != "gcp", + opts: []jobOption{withObjectLevelWI(objectLevelWIModeGCPImpersonateObj)}, + }, + { + name: "impersonation: object-level WIF direct access -> Impersonate (GCP)", + skip: !testImpersonation || *targetProvider != "gcp", + opts: []jobOption{withObjectLevelWI(objectLevelWIModeGCPImpersonateObjDA)}, + }, } { t.Run(tt.name, func(t *testing.T) { if tt.skip { diff --git a/tests/integration/restconfig_test.go b/tests/integration/restconfig_test.go index 150f3f6af..ac692048f 100644 --- a/tests/integration/restconfig_test.go +++ b/tests/integration/restconfig_test.go @@ -57,6 +57,41 @@ func TestRESTConfig(t *testing.T) { skip: !testWIDirectAccess || !testWIFederation, opts: []jobOption{withObjectLevelWI(objectLevelWIModeDirectAccessFederation)}, }, + { + name: "impersonation: controller IRSA -> AssumeRole (AWS)", + skip: !testImpersonation || *targetProvider != "aws", + opts: []jobOption{withObjectLevelWI(objectLevelWIModeAssumeRoleIRSA)}, + }, + { + name: "impersonation: controller Pod Identity -> AssumeRole (AWS)", + skip: !testImpersonation || *targetProvider != "aws", + opts: []jobOption{withObjectLevelWI(objectLevelWIModeAssumeRolePodIdentity)}, + }, + { + name: "impersonation: object-level IRSA -> AssumeRole (AWS)", + skip: !testImpersonation || *targetProvider != "aws", + opts: []jobOption{withObjectLevelWI(objectLevelWIModeAssumeRoleObjectLevel)}, + }, + { + name: "impersonation: controller WIF -> Impersonate (GCP)", + skip: !testImpersonation || *targetProvider != "gcp", + opts: []jobOption{withObjectLevelWI(objectLevelWIModeGCPImpersonateCtrl)}, + }, + { + name: "impersonation: controller WIF+SA -> Impersonate (GCP)", + skip: !testImpersonation || *targetProvider != "gcp", + opts: []jobOption{withObjectLevelWI(objectLevelWIModeGCPImpersonateCtrlSA)}, + }, + { + name: "impersonation: object-level WIF+SA -> Impersonate (GCP)", + skip: !testImpersonation || *targetProvider != "gcp", + opts: []jobOption{withObjectLevelWI(objectLevelWIModeGCPImpersonateObj)}, + }, + { + name: "impersonation: object-level WIF direct access -> Impersonate (GCP)", + skip: !testImpersonation || *targetProvider != "gcp", + opts: []jobOption{withObjectLevelWI(objectLevelWIModeGCPImpersonateObjDA)}, + }, } { t.Run(tt.name, func(t *testing.T) { if tt.skip { diff --git a/tests/integration/suite_test.go b/tests/integration/suite_test.go index 54ede5ac8..b52c666c7 100644 --- a/tests/integration/suite_test.go +++ b/tests/integration/suite_test.go @@ -112,6 +112,67 @@ const ( // workload identity. controllerWIRBACName = "flux-controller" + // wiControllerIRSA is the name of the controller service account with + // IRSA credentials for impersonation testing. + wiControllerIRSA = "flux-controller-irsa" + + // wiControllerPodIdentity is the name of the controller service account + // with Pod Identity credentials for impersonation testing. + wiControllerPodIdentity = "flux-controller-pod-identity" + + // wiControllerGCPSA is the name of the controller service account with + // GCP service account annotation for impersonation testing. + wiControllerGCPSA = "flux-controller-gcp-sa" + + // wiAssumeRoleCtrlSA is the name of the target SA for controller-level + // AWS AssumeRole impersonation (useServiceAccount: false). + wiAssumeRoleCtrlSA = "test-workload-id-assume-role-ctrl" + + // wiAssumeRoleSA is the name of the target SA for object-level + // AWS AssumeRole impersonation (useServiceAccount: true). + wiAssumeRoleSA = "test-workload-id-assume-role" + + // wiImpersonateCtrlSA is the name of the target SA for controller-level + // GCP impersonation (useServiceAccount: false). + wiImpersonateCtrlSA = "test-workload-id-impersonate-ctrl" + + // wiImpersonateSA is the name of the target SA for object-level GCP + // impersonation with GCP SA (useServiceAccount: true). + wiImpersonateSA = "test-workload-id-impersonate" + + // wiImpersonateDirectAccessSA is the name of the target SA for + // object-level GCP impersonation with direct access federation + // (useServiceAccount: true). + wiImpersonateDirectAccessSA = "test-workload-id-impersonate-da" + + // envVarWISANameAssumeRole is the name of the terraform environment + // variable for the assume-role SA name (object-level IRSA → AssumeRole). + envVarWISANameAssumeRole = "TF_VAR_wi_k8s_sa_name_assume_role" + + // envVarWISANameControllerIRSA is the name of the terraform environment + // variable for the controller IRSA SA name. + envVarWISANameControllerIRSA = "TF_VAR_wi_k8s_sa_name_controller_irsa" + + // envVarWISANameControllerPodIdentity is the name of the terraform environment + // variable for the controller Pod Identity SA name. + envVarWISANameControllerPodIdentity = "TF_VAR_wi_k8s_sa_name_controller_pod_identity" + + // envVarWISANameImpersonationTarget is the name of the terraform environment + // variable for the GCP impersonation target SA name (with GCP SA annotation). + envVarWISANameImpersonationTarget = "TF_VAR_wi_k8s_sa_name_impersonation_target" + + // envVarWISANameImpersonationDA is the name of the terraform environment + // variable for the GCP impersonation target SA name (with WIF direct access). + envVarWISANameImpersonationDA = "TF_VAR_wi_k8s_sa_name_impersonation_da" + + // envVarWISANameController is the name of the terraform environment + // variable for the default controller SA name (for GKE WIF principal). + envVarWISANameController = "TF_VAR_wi_k8s_sa_name_controller" + + // envVarWISANameControllerGCPSA is the name of the terraform environment + // variable for the controller SA with GCP SA annotation. + envVarWISANameControllerGCPSA = "TF_VAR_wi_k8s_sa_name_controller_gcp_sa" + // skippedMessage is the message used to skip tests for features // that are not supported by the provider or cluster configuration. skippedMessage = "Skipping test, feature not supported by the provider or by the current cluster configuration" @@ -172,6 +233,9 @@ var ( // testWIFederation is set by the provider config. testWIFederation bool + // testImpersonation is set when impersonation testing is enabled. + testImpersonation bool + // testGitCfg is a struct containing different variables needed for running git tests. testGitCfg *gitTestConfig @@ -217,6 +281,13 @@ type getGitTestConfig func(output map[string]*tfjson.StateOutput) (*gitTestConfi // loadGitSSHSecret is used to load the SSH key pair for git authentication. type loadGitSSHSecret func(output map[string]*tfjson.StateOutput) (map[string]string, string, error) +// getImpersonationAnnotations returns multiple sets of annotations for creating +// impersonation-related service accounts. The returned map is keyed by SA name. +type getImpersonationAnnotations func(output map[string]*tfjson.StateOutput) (map[string]map[string]string, error) + +// getControllerAnnotations returns annotations for a controller service account. +type getControllerAnnotations func(output map[string]*tfjson.StateOutput) (map[string]map[string]string, error) + // gitTestConfig hold different variable that will be needed by the different test functions. type gitTestConfig struct { // authentication info for git repositories @@ -270,6 +341,13 @@ type ProviderConfig struct { supportsGit bool // loadGitSSHSecret is used to load the SSH key pair for git authentication. loadGitSSHSecret loadGitSSHSecret + // supportsImpersonation indicates whether impersonation tests should run. + supportsImpersonation bool + // getImpersonationAnnotations returns annotations for impersonation target SAs. + getImpersonationAnnotations getImpersonationAnnotations + // getControllerAnnotations returns annotations for controller SAs + // used in impersonation testing. + getControllerAnnotations getControllerAnnotations } var letterRunes = []rune("abcdefghijklmnopqrstuvwxyz1234567890") @@ -300,6 +378,7 @@ func TestMain(m *testing.M) { if enableWI { testWIDirectAccess = providerCfg.supportsWIDirectAccess testWIFederation = providerCfg.supportsWIFederation + testImpersonation = providerCfg.supportsImpersonation // we only support git with workload identity testGit = providerCfg.supportsGit // we only support cluster auth with workload identity @@ -311,6 +390,13 @@ func TestMain(m *testing.M) { os.Setenv(envVarWISANameDirectAccess, wiServiceAccountDirectAccess) os.Setenv(envVarWISANameFederation, wiServiceAccountFederation) os.Setenv(envVarWISANameFederationDirectAccess, wiServiceAccountFederationDirectAccess) + os.Setenv(envVarWISANameAssumeRole, wiAssumeRoleSA) + os.Setenv(envVarWISANameControllerIRSA, wiControllerIRSA) + os.Setenv(envVarWISANameControllerPodIdentity, wiControllerPodIdentity) + os.Setenv(envVarWISANameImpersonationTarget, wiImpersonateSA) + os.Setenv(envVarWISANameImpersonationDA, wiImpersonateDirectAccessSA) + os.Setenv(envVarWISANameController, controllerWIRBACName) + os.Setenv(envVarWISANameControllerGCPSA, wiControllerGCPSA) // Run destroy-only mode if enabled. if *destroyOnly { @@ -432,6 +518,9 @@ func getProviderConfig(provider string) *ProviderConfig { grantPermissionsToGitRepository: grantPermissionsToGitRepositoryAWS, revokePermissionsToGitRepository: revokePermissionsToGitRepositoryAWS, getGitTestConfig: getGitTestConfigAWS, + supportsImpersonation: true, + getImpersonationAnnotations: getImpersonationAnnotationsAWS, + getControllerAnnotations: getControllerAnnotationsAWS, } case "azure": providerCfg := &ProviderConfig{ @@ -464,6 +553,9 @@ func getProviderConfig(provider string) *ProviderConfig { getGitTestConfig: getGitTestConfigGCP, supportsWIDirectAccess: true, supportsWIFederation: true, + supportsImpersonation: true, + getImpersonationAnnotations: getImpersonationAnnotationsGCP, + getControllerAnnotations: getControllerAnnotationsGCP, } } return nil @@ -629,6 +721,32 @@ func configureAdditionalInfra(ctx context.Context, providerCfg *ProviderConfig, if err := grantNamespaceAdminToClusterUsers(ctx, clusterUsers); err != nil { panic(err) } + + if testImpersonation { + log.Println("Impersonation is enabled, creating controller and target service accounts") + + // Create impersonation target SAs (provider-specific annotations). + impersonationAnnotations, err := providerCfg.getImpersonationAnnotations(tfOutput) + if err != nil { + panic(err) + } + for saName, saAnnotations := range impersonationAnnotations { + if err := createServiceAccountWithAnnotations(ctx, saName, saAnnotations); err != nil { + panic(fmt.Sprintf("failed to create impersonation SA %s: %v", saName, err)) + } + } + + // Create controller SAs for impersonation (provider-specific annotations). + controllerAnnotations, err := providerCfg.getControllerAnnotations(tfOutput) + if err != nil { + panic(err) + } + for saName, saAnnotations := range controllerAnnotations { + if err := createControllerServiceAccount(ctx, saName, saAnnotations); err != nil { + panic(fmt.Sprintf("failed to create controller SA %s: %v", saName, err)) + } + } + } } } @@ -805,6 +923,73 @@ func createControllerWorkloadIdentityServiceAccount(ctx context.Context) error { return nil } +// createServiceAccountWithAnnotations creates a service account with the given +// name and annotations in the default namespace. +func createServiceAccountWithAnnotations(ctx context.Context, name string, annotations map[string]string) error { + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: wiSANamespace, + }, + } + _, err := controllerutil.CreateOrUpdate(ctx, testEnv.Client, sa, func() error { + sa.Annotations = annotations + return nil + }) + if err != nil { + return fmt.Errorf("failed to create service account %s: %w", name, err) + } + return nil +} + +// createControllerServiceAccount creates a controller service account with the +// given name and annotations, and binds it to the existing controller ClusterRole +// so it can impersonate workload identity service accounts. +func createControllerServiceAccount(ctx context.Context, name string, annotations map[string]string) error { + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: wiSANamespace, + }, + } + _, err := controllerutil.CreateOrUpdate(ctx, testEnv.Client, sa, func() error { + sa.Annotations = annotations + return nil + }) + if err != nil { + return fmt.Errorf("failed to create controller service account %s: %w", name, err) + } + + // Bind the controller SA to the existing ClusterRole. + roleRef := rbacv1.RoleRef{ + APIGroup: rbacv1.SchemeGroupVersion.Group, + Kind: "ClusterRole", + Name: controllerWIRBACName, + } + subjects := []rbacv1.Subject{{ + Kind: "ServiceAccount", + Name: name, + Namespace: wiSANamespace, + }} + roleBinding := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + RoleRef: roleRef, + Subjects: subjects, + } + _, err = controllerutil.CreateOrUpdate(ctx, testEnv.Client, roleBinding, func() error { + roleBinding.RoleRef = roleRef + roleBinding.Subjects = subjects + return nil + }) + if err != nil { + return fmt.Errorf("failed to create controller cluster role binding for %s: %w", name, err) + } + + return nil +} + // createClusterConfigMapAndConfigureRBAC creates a configmap with the cluster // kubeconfig and configures RBAC to allow the test jobs to read it. func createClusterConfigMapAndConfigureRBAC(ctx context.Context, cmData map[string]string) error { @@ -860,6 +1045,21 @@ func createClusterConfigMapAndConfigureRBAC(ctx context.Context, cmData map[stri Name: controllerWIRBACName, Namespace: cm.Namespace, }, + { + Kind: "ServiceAccount", + Name: wiControllerIRSA, + Namespace: cm.Namespace, + }, + { + Kind: "ServiceAccount", + Name: wiControllerPodIdentity, + Namespace: cm.Namespace, + }, + { + Kind: "ServiceAccount", + Name: wiControllerGCPSA, + Namespace: cm.Namespace, + }, } roleBinding := &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ diff --git a/tests/integration/terraform/aws/main.tf b/tests/integration/terraform/aws/main.tf index 654356200..c2d84cabe 100644 --- a/tests/integration/terraform/aws/main.tf +++ b/tests/integration/terraform/aws/main.tf @@ -50,11 +50,22 @@ resource "aws_iam_role" "assume_role" { count = var.enable_wi ? 1 : 0 name = local.name description = "IAM role used for testing Workload integration for OCI repositories in Flux" - assume_role_policy = templatefile("oidc_assume_role_policy.json", { - OIDC_ARN = module.eks.cluster_oidc_arn, - OIDC_URL = replace(module.eks.cluster_oidc_url, "https://", ""), - NAMESPACE = var.wi_k8s_sa_ns, - SA_NAME = var.wi_k8s_sa_name + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Federated = module.eks.cluster_oidc_arn } + Action = "sts:AssumeRoleWithWebIdentity" + Condition = { + StringEquals = { + "${replace(module.eks.cluster_oidc_url, "https://", "")}:aud" = "sts.amazonaws.com" + "${replace(module.eks.cluster_oidc_url, "https://", "")}:sub" = [ + "system:serviceaccount:${var.wi_k8s_sa_ns}:${var.wi_k8s_sa_name}", + "system:serviceaccount:${var.wi_k8s_sa_ns}:${var.wi_k8s_sa_name_assume_role}", + ] + } + } + }] }) tags = var.tags @@ -103,3 +114,131 @@ resource "aws_eks_access_entry" "wi_access_entry" { principal_arn = aws_iam_role.assume_role[0].arn user_name = aws_iam_role.assume_role[0].arn } + +# --- Impersonation (AssumeRole) testing resources --- + +# Target IAM role that will be assumed via impersonation. +resource "aws_iam_role" "assume_role_target" { + count = var.enable_wi ? 1 : 0 + name = "${local.name}-target" + description = "Target IAM role for AssumeRole impersonation testing" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { AWS = [ + aws_iam_role.assume_role[0].arn, + aws_iam_role.controller_irsa[0].arn, + aws_iam_role.controller_pod_identity[0].arn, + ]} + Action = ["sts:AssumeRole", "sts:TagSession"] + } + ] + }) + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "target_read_ecr" { + count = var.enable_wi ? 1 : 0 + + role = aws_iam_role.assume_role_target[0].name + policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" +} + +resource "aws_iam_role_policy_attachment" "target_role_policy" { + count = var.enable_wi ? 1 : 0 + + role = aws_iam_role.assume_role_target[0].name + policy_arn = aws_iam_policy.wi_role_policy[0].arn +} + +resource "aws_eks_access_entry" "target_access_entry" { + count = var.enable_wi ? 1 : 0 + + depends_on = [module.eks] + + cluster_name = local.name + principal_arn = aws_iam_role.assume_role_target[0].arn + user_name = aws_iam_role.assume_role_target[0].arn +} + +# Managed policy granting sts:AssumeRole on the target role. +# Shared by all source roles that need to assume the target. +resource "aws_iam_policy" "assume_target_policy" { + count = var.enable_wi ? 1 : 0 + + name = "${local.name}-assume-target" + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Action = "sts:AssumeRole" + Resource = aws_iam_role.assume_role_target[0].arn + }] + }) +} + +# Allow the existing IRSA role to AssumeRole into the target. +resource "aws_iam_role_policy_attachment" "wi_assume_target" { + count = var.enable_wi ? 1 : 0 + + role = aws_iam_role.assume_role[0].name + policy_arn = aws_iam_policy.assume_target_policy[0].arn +} + +# Controller IRSA role — trusted by OIDC, only has sts:AssumeRole on target. +resource "aws_iam_role" "controller_irsa" { + count = var.enable_wi ? 1 : 0 + name = "${local.name}-ctrl-irsa" + description = "Controller IRSA role for AssumeRole impersonation testing" + assume_role_policy = templatefile("oidc_assume_role_policy.json", { + OIDC_ARN = module.eks.cluster_oidc_arn, + OIDC_URL = replace(module.eks.cluster_oidc_url, "https://", ""), + NAMESPACE = var.wi_k8s_sa_ns, + SA_NAME = var.wi_k8s_sa_name_controller_irsa + }) + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "controller_irsa_assume" { + count = var.enable_wi ? 1 : 0 + + role = aws_iam_role.controller_irsa[0].name + policy_arn = aws_iam_policy.assume_target_policy[0].arn +} + +# Controller Pod Identity role — trusted by EKS Pod Identity, only has sts:AssumeRole on target. +resource "aws_iam_role" "controller_pod_identity" { + count = var.enable_wi ? 1 : 0 + name = "${local.name}-ctrl-podid" + description = "Controller Pod Identity role for AssumeRole impersonation testing" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = "pods.eks.amazonaws.com" } + Action = ["sts:AssumeRole", "sts:TagSession"] + }] + }) + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "controller_pod_identity_assume" { + count = var.enable_wi ? 1 : 0 + + role = aws_iam_role.controller_pod_identity[0].name + policy_arn = aws_iam_policy.assume_target_policy[0].arn +} + +# Pod Identity association for the controller SA. +resource "aws_eks_pod_identity_association" "controller" { + count = var.enable_wi ? 1 : 0 + + depends_on = [module.eks] + + cluster_name = local.name + namespace = var.wi_k8s_sa_ns + service_account = var.wi_k8s_sa_name_controller_pod_identity + role_arn = aws_iam_role.controller_pod_identity[0].arn +} diff --git a/tests/integration/terraform/aws/outputs.tf b/tests/integration/terraform/aws/outputs.tf index 3a3f4a5ba..e7d90ec37 100644 --- a/tests/integration/terraform/aws/outputs.tf +++ b/tests/integration/terraform/aws/outputs.tf @@ -47,3 +47,11 @@ output "aws_wi_iam_arn" { output "ecrpublic_repository_url" { value = aws_ecrpublic_repository.test_ecr_public.repository_uri } + +output "aws_assume_role_target_arn" { + value = var.enable_wi ? aws_iam_role.assume_role_target[0].arn : "" +} + +output "aws_controller_irsa_arn" { + value = var.enable_wi ? aws_iam_role.controller_irsa[0].arn : "" +} diff --git a/tests/integration/terraform/aws/variables.tf b/tests/integration/terraform/aws/variables.tf index b6ac8defd..b2d2a6b32 100644 --- a/tests/integration/terraform/aws/variables.tf +++ b/tests/integration/terraform/aws/variables.tf @@ -22,6 +22,21 @@ variable "wi_k8s_sa_name" { description = "Name of kubernetes service account that can assume the IAM role (For workload identity)" } +variable "wi_k8s_sa_name_controller_irsa" { + type = string + description = "Name of controller SA with IRSA for impersonation testing" +} + +variable "wi_k8s_sa_name_controller_pod_identity" { + type = string + description = "Name of controller SA with Pod Identity for impersonation testing" +} + +variable "wi_k8s_sa_name_assume_role" { + type = string + description = "Name of SA with IRSA that will also AssumeRole into the target (object-level impersonation)" +} + variable "enable_wi" { type = bool default = false diff --git a/tests/integration/terraform/gcp/main.tf b/tests/integration/terraform/gcp/main.tf index 475152f36..0b5c228f8 100644 --- a/tests/integration/terraform/gcp/main.tf +++ b/tests/integration/terraform/gcp/main.tf @@ -7,7 +7,7 @@ provider "google" { resource "random_pet" "suffix" {} locals { - name = "flux-test-${random_pet.suffix.id}" + name = substr("flux-test-${random_pet.suffix.id}", 0, 25) federation_pool_id = var.enable_wi ? google_iam_workload_identity_pool.main[0].name : "" gke_pool_id = "projects/${data.google_project.project.number}/locations/global/workloadIdentityPools/${module.gke.project}.svc.id.goog" @@ -27,10 +27,14 @@ locals { # direct access. wi_k8s_sa_principal_direct_access_federation = var.enable_wi ? "principal://iam.googleapis.com/${local.federation_pool_id}/subject/system:serviceaccount:${var.wi_k8s_sa_ns}:${var.wi_k8s_sa_name_federation_direct_access}" : "" + # Principal for the impersonation target GCP SA. + wi_impersonation_target_principal = var.enable_wi ? "serviceAccount:${google_service_account.impersonation_target[0].email}" : "" + permission_principals = var.enable_wi ? [ local.wi_gcp_sa_principal, local.wi_k8s_sa_principal_direct_access, local.wi_k8s_sa_principal_direct_access_federation, + local.wi_impersonation_target_principal, ] : [] } @@ -91,6 +95,16 @@ resource "google_service_account_iam_binding" "main" { # testing Workload Identity Federation for other Kubernetes clusters with # impersonation. "principal://iam.googleapis.com/${local.federation_pool_id}/subject/system:serviceaccount:${var.wi_k8s_sa_ns}:${var.wi_k8s_sa_name_federation}", + + # This principal represents the impersonation target SA that needs to + # get the existing GCP SA's token as step 1 of GCP impersonation + # (GCPImpersonateObj: useServiceAccount=true with GCP SA annotation). + "serviceAccount:${var.gcp_project_id}.svc.id.goog[${var.wi_k8s_sa_ns}/${var.wi_k8s_sa_name_impersonation_target}]", + + # This principal represents the controller SA with GCP SA annotation + # that needs to get the existing GCP SA's token via GKE WIF + # (GCPImpersonateCtrlSA: controller with iam.gke.io/gcp-service-account). + "serviceAccount:${var.gcp_project_id}.svc.id.goog[${var.wi_k8s_sa_ns}/${var.wi_k8s_sa_name_controller_gcp_sa}]", ] } @@ -105,6 +119,39 @@ resource "google_service_account_iam_binding" "main" { # Workload Identity Pool and Provider as AWS EKS IRSA and built-in # Workload Identity Federation for GKE as AWS EKS Pod Identity). +# --- Impersonation testing resources --- + +# Impersonation target GCP service account. +resource "google_service_account" "impersonation_target" { + count = var.enable_wi ? 1 : 0 + account_id = "${local.name}-impt" + project = var.gcp_project_id + description = "Target service account for impersonation testing" +} + +# Grant serviceAccountTokenCreator on the target to principals that +# will impersonate it via the impersonate.CredentialsTokenSource API. +resource "google_service_account_iam_binding" "impersonation_token_creator" { + count = var.enable_wi ? 1 : 0 + service_account_id = google_service_account.impersonation_target[0].name + role = "roles/iam.serviceAccountTokenCreator" + members = [ + # Controller-level with GCP SA (GCPImpersonateCtrlSA + GCPImpersonateObj): + # the existing GCP SA identity. + local.wi_gcp_sa_principal, + + # Controller-level without GCP SA (GCPImpersonateCtrl): the controller + # K8s SA's GKE WIF federated identity. + "serviceAccount:${var.gcp_project_id}.svc.id.goog[${var.wi_k8s_sa_ns}/${var.wi_k8s_sa_name_controller}]", + + # Object-level WIF direct access (GCPImpersonateObjDA): the target SA's + # custom WIF pool federated identity. + "principal://iam.googleapis.com/${local.federation_pool_id}/subject/system:serviceaccount:${var.wi_k8s_sa_ns}:${var.wi_k8s_sa_name_impersonation_da}", + ] +} + +# --- Workload Identity Federation resources --- + resource "google_iam_workload_identity_pool" "main" { count = var.enable_wi ? 1 : 0 workload_identity_pool_id = local.name diff --git a/tests/integration/terraform/gcp/outputs.tf b/tests/integration/terraform/gcp/outputs.tf index fa49d25d2..5f77e0a8b 100644 --- a/tests/integration/terraform/gcp/outputs.tf +++ b/tests/integration/terraform/gcp/outputs.tf @@ -38,3 +38,7 @@ output "wi_k8s_sa_principal_direct_access" { output "wi_k8s_sa_principal_direct_access_federation" { value = var.enable_wi ? local.wi_k8s_sa_principal_direct_access_federation : "" } + +output "impersonation_target_email" { + value = var.enable_wi ? google_service_account.impersonation_target[0].email : "" +} diff --git a/tests/integration/terraform/gcp/variables.tf b/tests/integration/terraform/gcp/variables.tf index 9c1efb565..a85a2734f 100644 --- a/tests/integration/terraform/gcp/variables.tf +++ b/tests/integration/terraform/gcp/variables.tf @@ -42,6 +42,26 @@ variable "wi_k8s_sa_name_federation_direct_access" { description = "Name of kubernetes service account to get direct permissions in GCP (For workload identity federation)" } +variable "wi_k8s_sa_name_impersonation_target" { + type = string + description = "Name of kubernetes service account used as impersonation target with GCP SA annotation" +} + +variable "wi_k8s_sa_name_impersonation_da" { + type = string + description = "Name of kubernetes service account used as impersonation target with WIF direct access" +} + +variable "wi_k8s_sa_name_controller" { + type = string + description = "Name of the default controller kubernetes service account (for GKE WIF principal)" +} + +variable "wi_k8s_sa_name_controller_gcp_sa" { + type = string + description = "Name of the controller kubernetes service account with GCP SA annotation" +} + variable "enable_wi" { type = bool default = false