d8dfun/virtual-kubelet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

queue: Add tracing

Browse Source 
This adds tracing throughout the queues, so we can determine what's going on.
This commit is contained in:
Sargun Dhillon
2021-02-03 09:42:45 -08:00
parent ac9a1af564
commit 3a361ebabd
4 changed files with 98 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
internal/queue/queue.go
View File

@@ -100,39 +100,63 @@ func New(ratelimiter workqueue.RateLimiter, name string, handler ItemHandler) *Q
}
// Enqueue enqueues the key in a rate limited fashion
func (q *Queue) Enqueue(key string) {
func (q *Queue) Enqueue(ctx context.Context, key string) {
q.lock.Lock()
defer q.lock.Unlock()
q.insert(key, true, 0)
q.insert(ctx, key, true, 0)
}
// EnqueueWithoutRateLimit enqueues the key without a rate limit
func (q *Queue) EnqueueWithoutRateLimit(key string) {
func (q *Queue) EnqueueWithoutRateLimit(ctx context.Context, key string) {
q.lock.Lock()
defer q.lock.Unlock()
q.insert(key, false, 0)
q.insert(ctx, key, false, 0)
}
// Forget forgets the key
func (q *Queue) Forget(key string) {
func (q *Queue) Forget(ctx context.Context, key string) {
q.lock.Lock()
defer q.lock.Unlock()
ctx, span := trace.StartSpan(ctx, "Forget")
defer span.End()
ctx = span.WithFields(ctx, map[string]interface{}{
"queue": q.name,
"key": key,
})
if item, ok := q.itemsInQueue[key]; ok {
span.WithField(ctx, "status", "itemInQueue")
delete(q.itemsInQueue, key)
q.items.Remove(item)
return
}
if qi, ok := q.itemsBeingProcessed[key]; ok {
span.WithField(ctx, "status", "itemBeingProcessed")
qi.forget = true
return
}
span.WithField(ctx, "status", "notfound")
}
// insert inserts a new item to be processed at time time. It will not further delay items if when is later than the
// original time the item was scheduled to be processed. If when is earlier, it will "bring it forward"
func (q *Queue) insert(key string, ratelimit bool, delay time.Duration) *queueItem {
func (q *Queue) insert(ctx context.Context, key string, ratelimit bool, delay time.Duration) *queueItem {
ctx, span := trace.StartSpan(ctx, "insert")
defer span.End()
ctx = span.WithFields(ctx, map[string]interface{}{
"queue": q.name,
"key": key,
"ratelimit": ratelimit,
})
if delay > 0 {
ctx = span.WithField(ctx, "delay", delay.String())
}
defer func() {
select {
case q.wakeupCh <- struct{}{}:
@@ -142,6 +166,7 @@ func (q *Queue) insert(key string, ratelimit bool, delay time.Duration) *queueIt
// First see if the item is already being processed
if item, ok := q.itemsBeingProcessed[key]; ok {
span.WithField(ctx, "status", "itemsBeingProcessed")
when := q.clock.Now().Add(delay)
// Is the item already been redirtied?
if item.redirtiedAt.IsZero() {
@@ -157,12 +182,14 @@ func (q *Queue) insert(key string, ratelimit bool, delay time.Duration) *queueIt
// Is the item already in the queue?
if item, ok := q.itemsInQueue[key]; ok {
span.WithField(ctx, "status", "itemsInQueue")
qi := item.Value.(*queueItem)
when := q.clock.Now().Add(delay)
q.adjustPosition(qi, item, when)
return qi
}
span.WithField(ctx, "status", "added")
now := q.clock.Now()
val := &queueItem{
key: key,
@@ -175,6 +202,7 @@ func (q *Queue) insert(key string, ratelimit bool, delay time.Duration) *queueIt
panic("Non-zero delay with rate limiting not supported")
}
ratelimitDelay := q.ratelimiter.When(key)
span.WithField(ctx, "delay", ratelimitDelay.String())
val.plannedToStartWorkAt = val.plannedToStartWorkAt.Add(ratelimitDelay)
val.delayedViaRateLimit = &ratelimitDelay
} else {
@@ -213,10 +241,10 @@ func (q *Queue) adjustPosition(qi *queueItem, element *list.Element, when time.T
}
// EnqueueWithoutRateLimitWithDelay enqueues without rate limiting, but work will not start for this given delay period
func (q *Queue) EnqueueWithoutRateLimitWithDelay(key string, after time.Duration) {
func (q *Queue) EnqueueWithoutRateLimitWithDelay(ctx context.Context, key string, after time.Duration) {
q.lock.Lock()
defer q.lock.Unlock()
q.insert(key, false, after)
q.insert(ctx, key, false, after)
}
// Empty returns if the queue has no items in it
@@ -398,7 +426,7 @@ func (q *Queue) handleQueueItemObject(ctx context.Context, qi *queueItem) error
if qi.requeues+1 < MaxRetries {
// Put the item back on the work Queue to handle any transient errors.
log.G(ctx).WithError(err).Warnf("requeuing %q due to failed sync", qi.key)
newQI := q.insert(qi.key, true, 0)
newQI := q.insert(ctx, qi.key, true, 0)
newQI.requeues = qi.requeues + 1
newQI.originallyAdded = qi.originallyAdded
@@ -410,7 +438,7 @@ func (q *Queue) handleQueueItemObject(ctx context.Context, qi *queueItem) error
// We've exceeded the maximum retries or we were successful.
q.ratelimiter.Forget(qi.key)
if !qi.redirtiedAt.IsZero() {
newQI := q.insert(qi.key, qi.redirtiedWithRatelimit, time.Until(qi.redirtiedAt))
newQI := q.insert(ctx, qi.key, qi.redirtiedWithRatelimit, time.Until(qi.redirtiedAt))
newQI.addedViaRedirty = true
}

66
internal/queue/queue_test.go
View File

@@ -36,7 +36,7 @@ func TestQueueMaxRetries(t *testing.T) {
workqueue.NewItemExponentialFailureRateLimiter(5*time.Millisecond, 10*time.Millisecond),
&workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
), t.Name(), handler)
wq.Enqueue("test")
wq.Enqueue(context.TODO(), "test")
for n < MaxRetries {
assert.Assert(t, wq.handleQueueItem(ctx))
@@ -53,11 +53,11 @@ func TestForget(t *testing.T) {
}
wq := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), handler)
wq.Forget("val")
wq.Forget(context.TODO(), "val")
assert.Assert(t, is.Equal(0, wq.Len()))
v := "test"
wq.EnqueueWithoutRateLimit(v)
wq.EnqueueWithoutRateLimit(context.TODO(), v)
assert.Assert(t, is.Equal(1, wq.Len()))
}
@@ -86,8 +86,8 @@ func TestQueueItemNoSleep(t *testing.T) {
})
q.lock.Lock()
q.insert("foo", false, -1*time.Hour)
q.insert("bar", false, -1*time.Hour)
q.insert(ctx, "foo", false, -1*time.Hour)
q.insert(ctx, "bar", false, -1*time.Hour)
q.lock.Unlock()
item, err := q.getNextItem(ctx)
@@ -109,8 +109,8 @@ func TestQueueItemSleep(t *testing.T) {
return nil
})
q.lock.Lock()
q.insert("foo", false, 100*time.Millisecond)
q.insert("bar", false, 100*time.Millisecond)
q.insert(ctx, "foo", false, 100*time.Millisecond)
q.insert(ctx, "bar", false, 100*time.Millisecond)
q.lock.Unlock()
item, err := q.getNextItem(ctx)
@@ -131,7 +131,7 @@ func TestQueueBackgroundAdd(t *testing.T) {
time.AfterFunc(100*time.Millisecond, func() {
q.lock.Lock()
defer q.lock.Unlock()
q.insert("foo", false, 0)
q.insert(ctx, "foo", false, 0)
})
item, err := q.getNextItem(ctx)
@@ -151,13 +151,13 @@ func TestQueueBackgroundAdvance(t *testing.T) {
})
start := time.Now()
q.lock.Lock()
q.insert("foo", false, 10*time.Second)
q.insert(ctx, "foo", false, 10*time.Second)
q.lock.Unlock()
time.AfterFunc(200*time.Millisecond, func() {
q.lock.Lock()
defer q.lock.Unlock()
q.insert("foo", false, 0)
q.insert(ctx, "foo", false, 0)
})
item, err := q.getNextItem(ctx)
@@ -178,14 +178,14 @@ func TestQueueRedirty(t *testing.T) {
q = New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
assert.Assert(t, is.Equal(key, "foo"))
if atomic.AddInt64(&times, 1) == 1 {
q.EnqueueWithoutRateLimit("foo")
q.EnqueueWithoutRateLimit(context.TODO(), "foo")
} else {
cancel()
}
return nil
})
q.EnqueueWithoutRateLimit("foo")
q.EnqueueWithoutRateLimit(context.TODO(), "foo")
q.Run(ctx, 1)
for !q.Empty() {
time.Sleep(100 * time.Millisecond)
@@ -207,7 +207,7 @@ func TestHeapConcurrency(t *testing.T) {
return nil
})
for i := 0; i < 20; i++ {
q.EnqueueWithoutRateLimit(strconv.Itoa(i))
q.EnqueueWithoutRateLimit(context.TODO(), strconv.Itoa(i))
}
assert.Assert(t, q.Len() == 20)
@@ -241,20 +241,20 @@ func TestHeapOrder(t *testing.T) {
})
q.clock = nonmovingClock{}
q.EnqueueWithoutRateLimitWithDelay("a", 1000)
q.EnqueueWithoutRateLimitWithDelay("b", 2000)
q.EnqueueWithoutRateLimitWithDelay("c", 3000)
q.EnqueueWithoutRateLimitWithDelay("d", 4000)
q.EnqueueWithoutRateLimitWithDelay("e", 5000)
q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "a", 1000)
q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "b", 2000)
q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "c", 3000)
q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "d", 4000)
q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "e", 5000)
checkConsistency(t, q)
t.Logf("%v", q)
q.EnqueueWithoutRateLimitWithDelay("d", 1000)
q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "d", 1000)
checkConsistency(t, q)
t.Logf("%v", q)
q.EnqueueWithoutRateLimitWithDelay("c", 1001)
q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "c", 1001)
checkConsistency(t, q)
t.Logf("%v", q)
q.EnqueueWithoutRateLimitWithDelay("e", 999)
q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "e", 999)
checkConsistency(t, q)
t.Logf("%v", q)
}
@@ -316,7 +316,7 @@ func TestRateLimiter(t *testing.T) {
enqueued := 0
syncMap.Range(func(key, value interface{}) bool {
enqueued++
q.Enqueue(key.(string))
q.Enqueue(context.TODO(), key.(string))
return true
})
@@ -369,11 +369,11 @@ func TestQueueForgetInProgress(t *testing.T) {
q = New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
assert.Assert(t, is.Equal(key, "foo"))
atomic.AddInt64(&times, 1)
q.Forget(key)
q.Forget(context.TODO(), key)
return errors.New("test")
})
q.EnqueueWithoutRateLimit("foo")
q.EnqueueWithoutRateLimit(context.TODO(), "foo")
go q.Run(ctx, 1)
for !q.Empty() {
time.Sleep(100 * time.Millisecond)
@@ -390,8 +390,8 @@ func TestQueueForgetBeforeStart(t *testing.T) {
panic("shouldn't be called")
})
q.EnqueueWithoutRateLimit("foo")
q.Forget("foo")
q.EnqueueWithoutRateLimit(context.TODO(), "foo")
q.Forget(context.TODO(), "foo")
go q.Run(ctx, 1)
for !q.Empty() {
time.Sleep(100 * time.Millisecond)
@@ -401,26 +401,28 @@ func TestQueueForgetBeforeStart(t *testing.T) {
func TestQueueMoveItem(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
panic("shouldn't be called")
})
q.clock = nonmovingClock{}
q.insert("foo", false, 3000)
q.insert("bar", false, 2000)
q.insert("baz", false, 1000)
q.insert(ctx, "foo", false, 3000)
q.insert(ctx, "bar", false, 2000)
q.insert(ctx, "baz", false, 1000)
checkConsistency(t, q)
t.Log(q)
q.insert("foo", false, 2000)
q.insert(ctx, "foo", false, 2000)
checkConsistency(t, q)
t.Log(q)
q.insert("foo", false, 1999)
q.insert(ctx, "foo", false, 1999)
checkConsistency(t, q)
t.Log(q)
q.insert("foo", false, 999)
q.insert(ctx, "foo", false, 999)
checkConsistency(t, q)
t.Log(q)
}