Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ttl: fix some panic when delete a ttl timer manually (#57118) #57122

Merged
merged 1 commit into from
Nov 5, 2024
Merged

ttl: fix some panic when delete a ttl timer manually (#57118) #57122

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 16 additions & 7 deletions pkg/ttl/ttlworker/timer_sync.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,12 @@ func (g *TTLTimersSyncer) GetLastSyncInfo() (time.Time, int64) {
return g.lastSyncTime, g.lastSyncVer
}

// GetCachedTimerRecord returns a cached timer by key
func (g *TTLTimersSyncer) GetCachedTimerRecord(key string) (r *timerapi.TimerRecord, ok bool) {
r, ok = g.key2Timers[key]
return
}

// SyncTimers syncs timers with TTL tables
func (g *TTLTimersSyncer) SyncTimers(ctx context.Context, is infoschema.InfoSchema) {
g.lastSyncTime = g.nowFunc()
Expand Down Expand Up @@ -201,22 +207,25 @@ func (g *TTLTimersSyncer) SyncTimers(ctx context.Context, is infoschema.InfoSche
continue
}

timerID := timer.ID
if time.Since(timer.CreateTime) > g.delayDelete {
metrics.TTLSyncTimerCounter.Inc()
if _, err = g.cli.DeleteTimer(ctx, timer.ID); err != nil {
logutil.BgLogger().Error("failed to delete timer", zap.Error(err), zap.String("timerID", timer.ID))
if _, err = g.cli.DeleteTimer(ctx, timerID); err != nil {
logutil.BgLogger().Error("failed to delete timer", zap.Error(err), zap.String("timerID", timerID))
} else {
delete(g.key2Timers, key)
}
} else if timer.Enable {
metrics.TTLSyncTimerCounter.Inc()
if err = g.cli.UpdateTimer(ctx, timer.ID, timerapi.WithSetEnable(false)); err != nil {
logutil.BgLogger().Error("failed to disable timer", zap.Error(err), zap.String("timerID", timer.ID))
if err = g.cli.UpdateTimer(ctx, timerID, timerapi.WithSetEnable(false)); err != nil {
logutil.BgLogger().Error("failed to disable timer", zap.Error(err), zap.String("timerID", timerID))
}

timer, err = g.cli.GetTimerByID(ctx, timer.ID)
if err != nil {
logutil.BgLogger().Error("failed to get timer", zap.Error(err), zap.String("timerID", timer.ID))
timer, err = g.cli.GetTimerByID(ctx, timerID)
if errors.ErrorEqual(err, timerapi.ErrTimerNotExist) {
delete(g.key2Timers, key)
} else if err != nil {
logutil.BgLogger().Error("failed to get timer", zap.Error(err), zap.String("timerID", timerID))
} else {
g.key2Timers[key] = timer
}
Expand Down
21 changes: 18 additions & 3 deletions pkg/ttl/ttlworker/timer_sync_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -362,10 +362,25 @@ func TestTTLTimerSync(t *testing.T) {
time.Sleep(time.Second)
sync.SyncTimers(context.TODO(), do.InfoSchema())
require.Equal(t, syncCnt+8, syncTimerCounter.Val())
syncCnt = syncTimerCounter.Val()
checkTimerCnt(t, cli, 7)
checkTimersNotChange(t, cli, timer2, timer3, timer4, timer5, timerP10, timerP11, timerP12)

// https://github.com/pingcap/tidb/issues/57112
// table created and deleted but still in delay interval
timer3Key := timer3.Key
_, ok := sync.GetCachedTimerRecord(timer3Key)
require.True(t, ok)
sync.SetDelayDeleteInterval(time.Hour)
tk.MustExec("drop table t3")
tk.MustExec("delete from test.test_timers where ID = " + timer3.ID)
sync.SyncTimers(context.TODO(), do.InfoSchema())
require.Equal(t, syncCnt+9, syncTimerCounter.Val())
syncCnt = syncTimerCounter.Val()
// timer3 should be deleted from cache because the timer is deleted from storage.
_, ok = sync.GetCachedTimerRecord(timer3Key)
require.False(t, ok)
checkTimerCnt(t, cli, 6)

// reset timers
sync.Reset()
lastSyncTime, lastSyncVer = sync.GetLastSyncInfo()
Expand All @@ -381,8 +396,8 @@ func TestTTLTimerSync(t *testing.T) {
lastSyncTime, lastSyncVer = sync.GetLastSyncInfo()
require.Equal(t, do.InfoSchema().SchemaMetaVersion(), lastSyncVer)
require.GreaterOrEqual(t, lastSyncTime.Unix(), now.Unix())
checkTimerCnt(t, cli, 7)
checkTimersNotChange(t, cli, timer2, timer3, timer4, timer5, timerP10, timerP11, timerP12)
checkTimerCnt(t, cli, 6)
checkTimersNotChange(t, cli, timer2, timer4, timer5, timerP10, timerP11, timerP12)
}

func insertTTLTableStatusWatermark(t *testing.T, do *domain.Domain, tk *testkit.TestKit, db, table, partition string, watermark time.Time, jobRunning bool) {
Expand Down