This commit is contained in:
Pires
2023-03-13 10:46:29 +00:00
parent 7bcacb1cab
commit 63f85cc062
13 changed files with 982 additions and 880 deletions

View File

@@ -0,0 +1,206 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package token implements a manager of serviceaccount tokens for pods running
// on the node.
package token
import (
"context"
"errors"
"fmt"
"math/rand"
"sync"
"time"
authenticationv1 "k8s.io/api/authentication/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
"k8s.io/utils/clock"
)
const (
maxTTL = 24 * time.Hour
gcPeriod = time.Minute
maxJitter = 10 * time.Second
)
// NewManager returns a new token manager.
func NewManager(c clientset.Interface) *Manager {
// check whether the server supports token requests so we can give a more helpful error message
supported := false
once := &sync.Once{}
tokenRequestsSupported := func() bool {
once.Do(func() {
resources, err := c.Discovery().ServerResourcesForGroupVersion("v1")
if err != nil {
return
}
for _, resource := range resources.APIResources {
if resource.Name == "serviceaccounts/token" {
supported = true
return
}
}
})
return supported
}
m := &Manager{
getToken: func(name, namespace string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error) {
if c == nil {
return nil, errors.New("cannot use TokenManager when kubelet is in standalone mode")
}
tokenRequest, err := c.CoreV1().ServiceAccounts(namespace).CreateToken(context.TODO(), name, tr, metav1.CreateOptions{})
if apierrors.IsNotFound(err) && !tokenRequestsSupported() {
return nil, fmt.Errorf("the API server does not have TokenRequest endpoints enabled")
}
return tokenRequest, err
},
cache: make(map[string]*authenticationv1.TokenRequest),
clock: clock.RealClock{},
}
go wait.Forever(m.cleanup, gcPeriod)
return m
}
// Manager manages service account tokens for pods.
type Manager struct {
// cacheMutex guards the cache
cacheMutex sync.RWMutex
cache map[string]*authenticationv1.TokenRequest
// mocked for testing
getToken func(name, namespace string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error)
clock clock.Clock
}
// GetServiceAccountToken gets a service account token for a pod from cache or
// from the TokenRequest API. This process is as follows:
// * Check the cache for the current token request.
// * If the token exists and does not require a refresh, return the current token.
// * Attempt to refresh the token.
// * If the token is refreshed successfully, save it in the cache and return the token.
// * If refresh fails and the old token is still valid, log an error and return the old token.
// * If refresh fails and the old token is no longer valid, return an error
func (m *Manager) GetServiceAccountToken(namespace, name string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error) {
key := keyFunc(name, namespace, tr)
ctr, ok := m.get(key)
if ok && !m.requiresRefresh(ctr) {
return ctr, nil
}
tr, err := m.getToken(name, namespace, tr)
if err != nil {
switch {
case !ok:
return nil, fmt.Errorf("failed to fetch token: %v", err)
case m.expired(ctr):
return nil, fmt.Errorf("token %s expired and refresh failed: %v", key, err)
default:
klog.ErrorS(err, "Couldn't update token", "cacheKey", key)
return ctr, nil
}
}
m.set(key, tr)
return tr, nil
}
// DeleteServiceAccountToken should be invoked when pod got deleted. It simply
// clean token manager cache.
func (m *Manager) DeleteServiceAccountToken(podUID types.UID) {
m.cacheMutex.Lock()
defer m.cacheMutex.Unlock()
for k, tr := range m.cache {
if tr.Spec.BoundObjectRef.UID == podUID {
delete(m.cache, k)
}
}
}
func (m *Manager) cleanup() {
m.cacheMutex.Lock()
defer m.cacheMutex.Unlock()
for k, tr := range m.cache {
if m.expired(tr) {
delete(m.cache, k)
}
}
}
func (m *Manager) get(key string) (*authenticationv1.TokenRequest, bool) {
m.cacheMutex.RLock()
defer m.cacheMutex.RUnlock()
ctr, ok := m.cache[key]
return ctr, ok
}
func (m *Manager) set(key string, tr *authenticationv1.TokenRequest) {
m.cacheMutex.Lock()
defer m.cacheMutex.Unlock()
m.cache[key] = tr
}
func (m *Manager) expired(t *authenticationv1.TokenRequest) bool {
return m.clock.Now().After(t.Status.ExpirationTimestamp.Time)
}
// requiresRefresh returns true if the token is older than 80% of its total
// ttl, or if the token is older than 24 hours.
func (m *Manager) requiresRefresh(tr *authenticationv1.TokenRequest) bool {
if tr.Spec.ExpirationSeconds == nil {
cpy := tr.DeepCopy()
cpy.Status.Token = ""
klog.ErrorS(nil, "Expiration seconds was nil for token request", "tokenRequest", cpy)
return false
}
now := m.clock.Now()
exp := tr.Status.ExpirationTimestamp.Time
iat := exp.Add(-1 * time.Duration(*tr.Spec.ExpirationSeconds) * time.Second)
jitter := time.Duration(rand.Float64()*maxJitter.Seconds()) * time.Second
if now.After(iat.Add(maxTTL - jitter)) {
return true
}
// Require a refresh if within 20% of the TTL plus a jitter from the expiration time.
if now.After(exp.Add(-1*time.Duration((*tr.Spec.ExpirationSeconds*20)/100)*time.Second - jitter)) {
return true
}
return false
}
// keys should be nonconfidential and safe to log
func keyFunc(name, namespace string, tr *authenticationv1.TokenRequest) string {
var exp int64
if tr.Spec.ExpirationSeconds != nil {
exp = *tr.Spec.ExpirationSeconds
}
var ref authenticationv1.BoundObjectReference
if tr.Spec.BoundObjectRef != nil {
ref = *tr.Spec.BoundObjectRef
}
return fmt.Sprintf("%q/%q/%#v/%#v/%#v", name, namespace, tr.Spec.Audiences, exp, ref)
}

View File

@@ -0,0 +1,606 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package token
import (
"fmt"
"testing"
"time"
authenticationv1 "k8s.io/api/authentication/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
testingclock "k8s.io/utils/clock/testing"
)
func TestTokenCachingAndExpiration(t *testing.T) {
type suite struct {
clock *testingclock.FakeClock
tg *fakeTokenGetter
mgr *Manager
}
cases := []struct {
name string
exp time.Duration
f func(t *testing.T, s *suite)
}{
{
name: "rotate hour token expires in the last 12 minutes",
exp: time.Hour,
f: func(t *testing.T, s *suite) {
s.clock.SetTime(s.clock.Now().Add(50 * time.Minute))
if _, err := s.mgr.GetServiceAccountToken("a", "b", getTokenRequest()); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if s.tg.count != 2 {
t.Fatalf("expected token to be refreshed: call count was %d", s.tg.count)
}
},
},
{
name: "rotate 24 hour token that expires in 40 hours",
exp: 40 * time.Hour,
f: func(t *testing.T, s *suite) {
s.clock.SetTime(s.clock.Now().Add(25 * time.Hour))
if _, err := s.mgr.GetServiceAccountToken("a", "b", getTokenRequest()); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if s.tg.count != 2 {
t.Fatalf("expected token to be refreshed: call count was %d", s.tg.count)
}
},
},
{
name: "rotate hour token fails, old token is still valid, doesn't error",
exp: time.Hour,
f: func(t *testing.T, s *suite) {
s.clock.SetTime(s.clock.Now().Add(50 * time.Minute))
tg := &fakeTokenGetter{
err: fmt.Errorf("err"),
}
s.mgr.getToken = tg.getToken
tr, err := s.mgr.GetServiceAccountToken("a", "b", getTokenRequest())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if tr.Status.Token != "foo" {
t.Fatalf("unexpected token: %v", tr.Status.Token)
}
},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
clock := testingclock.NewFakeClock(time.Time{}.Add(30 * 24 * time.Hour))
expSecs := int64(c.exp.Seconds())
s := &suite{
clock: clock,
mgr: NewManager(nil),
tg: &fakeTokenGetter{
tr: &authenticationv1.TokenRequest{
Spec: authenticationv1.TokenRequestSpec{
ExpirationSeconds: &expSecs,
},
Status: authenticationv1.TokenRequestStatus{
Token: "foo",
ExpirationTimestamp: metav1.Time{Time: clock.Now().Add(c.exp)},
},
},
},
}
s.mgr.getToken = s.tg.getToken
s.mgr.clock = s.clock
if _, err := s.mgr.GetServiceAccountToken("a", "b", getTokenRequest()); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if s.tg.count != 1 {
t.Fatalf("unexpected client call, got: %d, want: 1", s.tg.count)
}
if _, err := s.mgr.GetServiceAccountToken("a", "b", getTokenRequest()); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if s.tg.count != 1 {
t.Fatalf("expected token to be served from cache: saw %d", s.tg.count)
}
c.f(t, s)
})
}
}
func TestRequiresRefresh(t *testing.T) {
start := time.Now()
cases := []struct {
now, exp time.Time
expectRefresh bool
requestTweaks func(*authenticationv1.TokenRequest)
}{
{
now: start.Add(10 * time.Minute),
exp: start.Add(60 * time.Minute),
expectRefresh: false,
},
{
now: start.Add(50 * time.Minute),
exp: start.Add(60 * time.Minute),
expectRefresh: true,
},
{
now: start.Add(25 * time.Hour),
exp: start.Add(60 * time.Hour),
expectRefresh: true,
},
{
now: start.Add(70 * time.Minute),
exp: start.Add(60 * time.Minute),
expectRefresh: true,
},
{
// expiry will be overwritten by the tweak below.
now: start.Add(0 * time.Minute),
exp: start.Add(60 * time.Minute),
expectRefresh: false,
requestTweaks: func(tr *authenticationv1.TokenRequest) {
tr.Spec.ExpirationSeconds = nil
},
},
}
for i, c := range cases {
t.Run(fmt.Sprint(i), func(t *testing.T) {
clock := testingclock.NewFakeClock(c.now)
secs := int64(c.exp.Sub(start).Seconds())
tr := &authenticationv1.TokenRequest{
Spec: authenticationv1.TokenRequestSpec{
ExpirationSeconds: &secs,
},
Status: authenticationv1.TokenRequestStatus{
ExpirationTimestamp: metav1.Time{Time: c.exp},
},
}
if c.requestTweaks != nil {
c.requestTweaks(tr)
}
mgr := NewManager(nil)
mgr.clock = clock
rr := mgr.requiresRefresh(tr)
if rr != c.expectRefresh {
t.Fatalf("unexpected requiresRefresh result, got: %v, want: %v", rr, c.expectRefresh)
}
})
}
}
func TestDeleteServiceAccountToken(t *testing.T) {
type request struct {
name, namespace string
tr authenticationv1.TokenRequest
shouldFail bool
}
cases := []struct {
name string
requestIndex []int
deletePodUID []types.UID
expLeftIndex []int
}{
{
name: "delete none with all success requests",
requestIndex: []int{0, 1, 2},
expLeftIndex: []int{0, 1, 2},
},
{
name: "delete one with all success requests",
requestIndex: []int{0, 1, 2},
deletePodUID: []types.UID{"fake-uid-1"},
expLeftIndex: []int{1, 2},
},
{
name: "delete two with all success requests",
requestIndex: []int{0, 1, 2},
deletePodUID: []types.UID{"fake-uid-1", "fake-uid-3"},
expLeftIndex: []int{1},
},
{
name: "delete all with all success requests",
requestIndex: []int{0, 1, 2},
deletePodUID: []types.UID{"fake-uid-1", "fake-uid-2", "fake-uid-3"},
},
{
name: "delete no pod with failed requests",
requestIndex: []int{0, 1, 2, 3},
deletePodUID: []types.UID{},
expLeftIndex: []int{0, 1, 2},
},
{
name: "delete other pod with failed requests",
requestIndex: []int{0, 1, 2, 3},
deletePodUID: []types.UID{"fake-uid-2"},
expLeftIndex: []int{0, 2},
},
{
name: "delete no pod with request which success after failure",
requestIndex: []int{0, 1, 2, 3, 4},
deletePodUID: []types.UID{},
expLeftIndex: []int{0, 1, 2, 4},
},
{
name: "delete the pod which success after failure",
requestIndex: []int{0, 1, 2, 3, 4},
deletePodUID: []types.UID{"fake-uid-4"},
expLeftIndex: []int{0, 1, 2},
},
{
name: "delete other pod with request which success after failure",
requestIndex: []int{0, 1, 2, 3, 4},
deletePodUID: []types.UID{"fake-uid-1"},
expLeftIndex: []int{1, 2, 4},
},
{
name: "delete some pod not in the set",
requestIndex: []int{0, 1, 2},
deletePodUID: []types.UID{"fake-uid-100", "fake-uid-200"},
expLeftIndex: []int{0, 1, 2},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
requests := []request{
{
name: "fake-name-1",
namespace: "fake-namespace-1",
tr: authenticationv1.TokenRequest{
Spec: authenticationv1.TokenRequestSpec{
BoundObjectRef: &authenticationv1.BoundObjectReference{
UID: "fake-uid-1",
Name: "fake-name-1",
},
},
},
shouldFail: false,
},
{
name: "fake-name-2",
namespace: "fake-namespace-2",
tr: authenticationv1.TokenRequest{
Spec: authenticationv1.TokenRequestSpec{
BoundObjectRef: &authenticationv1.BoundObjectReference{
UID: "fake-uid-2",
Name: "fake-name-2",
},
},
},
shouldFail: false,
},
{
name: "fake-name-3",
namespace: "fake-namespace-3",
tr: authenticationv1.TokenRequest{
Spec: authenticationv1.TokenRequestSpec{
BoundObjectRef: &authenticationv1.BoundObjectReference{
UID: "fake-uid-3",
Name: "fake-name-3",
},
},
},
shouldFail: false,
},
{
name: "fake-name-4",
namespace: "fake-namespace-4",
tr: authenticationv1.TokenRequest{
Spec: authenticationv1.TokenRequestSpec{
BoundObjectRef: &authenticationv1.BoundObjectReference{
UID: "fake-uid-4",
Name: "fake-name-4",
},
},
},
shouldFail: true,
},
{
//exactly the same with last one, besides it will success
name: "fake-name-4",
namespace: "fake-namespace-4",
tr: authenticationv1.TokenRequest{
Spec: authenticationv1.TokenRequestSpec{
BoundObjectRef: &authenticationv1.BoundObjectReference{
UID: "fake-uid-4",
Name: "fake-name-4",
},
},
},
shouldFail: false,
},
}
testMgr := NewManager(nil)
testMgr.clock = testingclock.NewFakeClock(time.Time{}.Add(30 * 24 * time.Hour))
successGetToken := func(_, _ string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error) {
tr.Status = authenticationv1.TokenRequestStatus{
ExpirationTimestamp: metav1.Time{Time: testMgr.clock.Now().Add(10 * time.Hour)},
}
return tr, nil
}
failGetToken := func(_, _ string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error) {
return nil, fmt.Errorf("fail tr")
}
for _, index := range c.requestIndex {
req := requests[index]
if req.shouldFail {
testMgr.getToken = failGetToken
} else {
testMgr.getToken = successGetToken
}
testMgr.GetServiceAccountToken(req.namespace, req.name, &req.tr)
}
for _, uid := range c.deletePodUID {
testMgr.DeleteServiceAccountToken(uid)
}
if len(c.expLeftIndex) != len(testMgr.cache) {
t.Errorf("%s got unexpected result: expected left cache size is %d, got %d", c.name, len(c.expLeftIndex), len(testMgr.cache))
}
for _, leftIndex := range c.expLeftIndex {
r := requests[leftIndex]
_, ok := testMgr.get(keyFunc(r.name, r.namespace, &r.tr))
if !ok {
t.Errorf("%s got unexpected result: expected token request %v exist in cache, but not", c.name, r)
}
}
})
}
}
type fakeTokenGetter struct {
count int
tr *authenticationv1.TokenRequest
err error
}
func (ftg *fakeTokenGetter) getToken(name, namespace string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error) {
ftg.count++
return ftg.tr, ftg.err
}
func TestCleanup(t *testing.T) {
cases := []struct {
name string
relativeExp time.Duration
expectedCacheSize int
}{
{
name: "don't cleanup unexpired tokens",
relativeExp: -1 * time.Hour,
expectedCacheSize: 0,
},
{
name: "cleanup expired tokens",
relativeExp: time.Hour,
expectedCacheSize: 1,
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
clock := testingclock.NewFakeClock(time.Time{}.Add(24 * time.Hour))
mgr := NewManager(nil)
mgr.clock = clock
mgr.set("key", &authenticationv1.TokenRequest{
Status: authenticationv1.TokenRequestStatus{
ExpirationTimestamp: metav1.Time{Time: mgr.clock.Now().Add(c.relativeExp)},
},
})
mgr.cleanup()
if got, want := len(mgr.cache), c.expectedCacheSize; got != want {
t.Fatalf("unexpected number of cache entries after cleanup, got: %d, want: %d", got, want)
}
})
}
}
func TestKeyFunc(t *testing.T) {
type tokenRequestUnit struct {
name string
namespace string
tr *authenticationv1.TokenRequest
}
getKeyFunc := func(u tokenRequestUnit) string {
return keyFunc(u.name, u.namespace, u.tr)
}
cases := []struct {
name string
trus []tokenRequestUnit
target tokenRequestUnit
shouldHit bool
}{
{
name: "hit",
trus: []tokenRequestUnit{
{
name: "foo-sa",
namespace: "foo-ns",
tr: &authenticationv1.TokenRequest{
Spec: authenticationv1.TokenRequestSpec{
Audiences: []string{"foo1", "foo2"},
ExpirationSeconds: getInt64Point(2000),
BoundObjectRef: &authenticationv1.BoundObjectReference{
Kind: "pod",
Name: "foo-pod",
UID: "foo-uid",
},
},
},
},
{
name: "ame-sa",
namespace: "ame-ns",
tr: &authenticationv1.TokenRequest{
Spec: authenticationv1.TokenRequestSpec{
Audiences: []string{"ame1", "ame2"},
ExpirationSeconds: getInt64Point(2000),
BoundObjectRef: &authenticationv1.BoundObjectReference{
Kind: "pod",
Name: "ame-pod",
UID: "ame-uid",
},
},
},
},
},
target: tokenRequestUnit{
name: "foo-sa",
namespace: "foo-ns",
tr: &authenticationv1.TokenRequest{
Spec: authenticationv1.TokenRequestSpec{
Audiences: []string{"foo1", "foo2"},
ExpirationSeconds: getInt64Point(2000),
BoundObjectRef: &authenticationv1.BoundObjectReference{
Kind: "pod",
Name: "foo-pod",
UID: "foo-uid",
},
},
},
},
shouldHit: true,
},
{
name: "not hit due to different ExpirationSeconds",
trus: []tokenRequestUnit{
{
name: "foo-sa",
namespace: "foo-ns",
tr: &authenticationv1.TokenRequest{
Spec: authenticationv1.TokenRequestSpec{
Audiences: []string{"foo1", "foo2"},
ExpirationSeconds: getInt64Point(2000),
BoundObjectRef: &authenticationv1.BoundObjectReference{
Kind: "pod",
Name: "foo-pod",
UID: "foo-uid",
},
},
},
},
},
target: tokenRequestUnit{
name: "foo-sa",
namespace: "foo-ns",
tr: &authenticationv1.TokenRequest{
Spec: authenticationv1.TokenRequestSpec{
Audiences: []string{"foo1", "foo2"},
//everthing is same besides ExpirationSeconds
ExpirationSeconds: getInt64Point(2001),
BoundObjectRef: &authenticationv1.BoundObjectReference{
Kind: "pod",
Name: "foo-pod",
UID: "foo-uid",
},
},
},
},
shouldHit: false,
},
{
name: "not hit due to different BoundObjectRef",
trus: []tokenRequestUnit{
{
name: "foo-sa",
namespace: "foo-ns",
tr: &authenticationv1.TokenRequest{
Spec: authenticationv1.TokenRequestSpec{
Audiences: []string{"foo1", "foo2"},
ExpirationSeconds: getInt64Point(2000),
BoundObjectRef: &authenticationv1.BoundObjectReference{
Kind: "pod",
Name: "foo-pod",
UID: "foo-uid",
},
},
},
},
},
target: tokenRequestUnit{
name: "foo-sa",
namespace: "foo-ns",
tr: &authenticationv1.TokenRequest{
Spec: authenticationv1.TokenRequestSpec{
Audiences: []string{"foo1", "foo2"},
ExpirationSeconds: getInt64Point(2000),
BoundObjectRef: &authenticationv1.BoundObjectReference{
Kind: "pod",
//everthing is same besides BoundObjectRef.Name
Name: "diff-pod",
UID: "foo-uid",
},
},
},
},
shouldHit: false,
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
mgr := NewManager(nil)
mgr.clock = testingclock.NewFakeClock(time.Time{}.Add(30 * 24 * time.Hour))
for _, tru := range c.trus {
mgr.set(getKeyFunc(tru), &authenticationv1.TokenRequest{
Status: authenticationv1.TokenRequestStatus{
//make sure the token cache would not be cleaned by token manager clenaup func
ExpirationTimestamp: metav1.Time{Time: mgr.clock.Now().Add(50 * time.Minute)},
},
})
}
_, hit := mgr.get(getKeyFunc(c.target))
if hit != c.shouldHit {
t.Errorf("%s got unexpected hit result: expected to be %t, got %t", c.name, c.shouldHit, hit)
}
})
}
}
func getTokenRequest() *authenticationv1.TokenRequest {
return &authenticationv1.TokenRequest{
Spec: authenticationv1.TokenRequestSpec{
Audiences: []string{"foo1", "foo2"},
ExpirationSeconds: getInt64Point(2000),
BoundObjectRef: &authenticationv1.BoundObjectReference{
Kind: "pod",
Name: "foo-pod",
UID: "foo-uid",
},
},
}
}
func getInt64Point(v int64) *int64 {
return &v
}