Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Restrict segment metadata kill query till maxInterval from last kill task time #17770

Merged
merged 4 commits into from
Mar 4, 2025
Merged

Restrict segment metadata kill query till maxInterval from last kill task time #17770

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
00bf784
Restrict segment metadata kill query till maxInterval from last kill …
chetanpatidar26 Mar 1, 2025
2ee9a84
fix variables
chetanpatidar26 Mar 3, 2025
3a3a847
fix variables
chetanpatidar26 Mar 3, 2025
91a8921
addressed comments
chetanpatidar26 Mar 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion server/src/main/java/org/apache/druid/server/coordinator/duty/KillUnusedSegments.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,9 +273,17 @@ private Interval findIntervalForKill(
)
{
final DateTime minStartTime = datasourceToLastKillIntervalEnd.get(dataSource);

// Once the first segment from a datasource is killed, we have a valid minStartTime.
// Restricting the upper bound to scan segments metadata while running the kill task results in a efficient SQL query.
final DateTime maxIntervalFromLastKill = minStartTime != null
? DateTimes.min(minStartTime.plus(maxIntervalToKill),
DateTimes.nowUtc().minus(durationToRetain))
: DateTimes.nowUtc().minus(durationToRetain);

final DateTime maxEndTime = ignoreDurationToRetain
? DateTimes.COMPARE_DATE_AS_STRING_MAX
: DateTimes.nowUtc().minus(durationToRetain);
: maxIntervalFromLastKill;

final List<Interval> unusedSegmentIntervals = limitToPeriod(
segmentsMetadataManager.getUnusedSegmentIntervals(
Expand Down
27 changes: 27 additions & 0 deletions server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ public class KillUnusedSegmentsTest
private static final DateTime NOW = DateTimes.nowUtc();
private static final Interval YEAR_OLD = new Interval(Period.days(1), NOW.minusDays(365));
private static final Interval MONTH_OLD = new Interval(Period.days(1), NOW.minusDays(30));
private static final Interval FIFTEEN_DAY_OLD = new Interval(Period.days(1), NOW.minusDays(15));
private static final Interval DAY_OLD = new Interval(Period.days(1), NOW.minusDays(1));
private static final Interval HOUR_OLD = new Interval(Period.days(1), NOW.minusHours(1));
private static final Interval NEXT_DAY = new Interval(Period.days(1), NOW.plusDays(1));
Expand All @@ -87,6 +88,9 @@ public class KillUnusedSegmentsTest
private static final String DS2 = "DS2";
private static final String DS3 = "DS3";

private static final Period RETAIN_DURATION = Period.hours(6);
private static final Period MAX_KILL_INTERVAL = Period.days(20);

private static final RowKey DS1_STAT_KEY = RowKey.of(Dimension.DATASOURCE, DS1);
private static final RowKey DS2_STAT_KEY = RowKey.of(Dimension.DATASOURCE, DS2);
private static final RowKey DS3_STAT_KEY = RowKey.of(Dimension.DATASOURCE, DS3);
Expand Down Expand Up @@ -604,6 +608,29 @@ public void testLowerMaxIntervalToKill()
validateLastKillStateAndReset(DS1, YEAR_OLD);
}

@Test
public void testRestrictKillQueryToMaxInterval()
{
configBuilder.withDurationToRetain(RETAIN_DURATION.toStandardDuration())
.withMaxIntervalToKill(MAX_KILL_INTERVAL);

initDuty();

createAndAddUnusedSegment(DS1, MONTH_OLD, VERSION, NOW.minusDays(29));
CoordinatorRunStats newDatasourceStats = runDutyAndGetStats();

Assert.assertEquals(1, newDatasourceStats.get(Stats.Kill.ELIGIBLE_UNUSED_SEGMENTS, DS1_STAT_KEY));
validateLastKillStateAndReset(DS1, MONTH_OLD);

createAndAddUnusedSegment(DS1, FIFTEEN_DAY_OLD, VERSION, NOW.minusDays(14));
createAndAddUnusedSegment(DS1, DAY_OLD, VERSION, NOW.minusHours(2));
CoordinatorRunStats oldDatasourceStats = runDutyAndGetStats();

Assert.assertEquals(2, oldDatasourceStats.get(Stats.Kill.ELIGIBLE_UNUSED_SEGMENTS, DS1_STAT_KEY));
validateLastKillStateAndReset(DS1, FIFTEEN_DAY_OLD);
}


@Test
public void testHigherMaxIntervalToKill()
{
Expand Down