Repository navigation
[core] Expire TIMESTAMP partitions without time zone with the values-time strategy #10851
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
8f80852
39bc84f
d3518fe
b79af8f
ed26257
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -34,7 +34,6 @@ | |
| import javax.annotation.Nullable; | ||
|
|
||
| import java.time.Duration; | ||
| import java.time.LocalDate; | ||
| import java.time.LocalDateTime; | ||
| import java.util.ArrayList; | ||
| import java.util.Collection; | ||
|
|
@@ -157,63 +156,87 @@ private List<Map<String, String>> doExpire( | |
| LocalDateTime expireDateTime, long commitIdentifier) { | ||
| List<PartitionEntry> partitionEntries = | ||
| strategy.selectExpiredPartitions(scan, expireDateTime); | ||
| List<List<String>> expiredPartValues = new ArrayList<>(partitionEntries.size()); | ||
| List<ExpiredPartition> expiredPartitions = new ArrayList<>(partitionEntries.size()); | ||
| for (PartitionEntry partition : partitionEntries) { | ||
| Object[] array = strategy.convertPartition(partition.partition()); | ||
| // DATE values are normalized for time extraction; use the stored names for dropping. | ||
| // DATE and TIMESTAMP values are normalized for time extraction; report the stored | ||
| // names instead. | ||
| Map<String, String> partitionSpec = | ||
| commit.pathFactory() | ||
| .partitionComputer() | ||
| .generatePartValues(partition.partition()); | ||
| int index = 0; | ||
| for (String value : partitionSpec.values()) { | ||
| if (array[index] instanceof LocalDate) { | ||
| if (array[index] != null && strategy.isNormalizedForTimeExtraction(index)) { | ||
| array[index] = value; | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [P1] Preserve the selected typed partition key when deleting locally. The generated path spelling is not always safe to parse back as a partition spec. With current names,
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Rechecked b79af8f: the new binary-key deletion fixes the local path, and 156 expiration/extractor/commit tests pass with normal JDK 8 Maven checks. The metastore path still has the same P1: This reproduces using the PR's own new tests: set I also ran the epoch/NULL case with the three expiration production files restored to merge-base d7b521d: it deletes epoch and retains NULL. The identical data assertion fails on this head (only the returned-name assertion is omitted in both runs because base reports ISO and head reports SQL). Please propagate the exact selected typed key through the metastore-backed data deletion path too, while retaining stored names for HMS metadata operations; selecting the right key only in the local branch leaves this regression active. |
||
| } | ||
| index++; | ||
| } | ||
| expiredPartValues.add(strategy.toPartitionValue(array)); | ||
| List<String> values = strategy.toPartitionValue(array); | ||
| // Sort on the joined form, but build the partition from the original values: a | ||
| // value may itself contain DELIMITER, and splitting the joined string back apart | ||
| // would then shift every following field by one. | ||
| expiredPartitions.add( | ||
| new ExpiredPartition( | ||
| partition.partition(), | ||
| String.join(DELIMITER, values), | ||
| strategy.toPartitionString(values.toArray()))); | ||
| } | ||
|
|
||
| List<Map<String, String>> expired = new ArrayList<>(); | ||
| if (!expiredPartValues.isEmpty()) { | ||
| // convert partition value to partition string, and limit the partition num | ||
| expired = convertToPartitionString(expiredPartValues); | ||
| List<ExpiredPartition> selected = | ||
| expiredPartitions.stream() | ||
| .sorted(Comparator.comparing(p -> p.sortKey)) | ||
| .limit(Math.min(expiredPartitions.size(), maxExpireNum)) | ||
| .collect(Collectors.toList()); | ||
| List<Map<String, String>> expired = | ||
| selected.stream().map(p -> p.name).collect(Collectors.toList()); | ||
| if (!selected.isEmpty()) { | ||
| LOG.info("Expire Partitions: {}", expired); | ||
| if (expireBatchSize > 0 && expireBatchSize < expired.size()) { | ||
| Lists.partition(expired, expireBatchSize) | ||
| if (expireBatchSize > 0 && expireBatchSize < selected.size()) { | ||
| Lists.partition(selected, expireBatchSize) | ||
| .forEach( | ||
| expiredBatchPartitions -> | ||
| doBatchExpire(expiredBatchPartitions, commitIdentifier)); | ||
| } else { | ||
| doBatchExpire(expired, commitIdentifier); | ||
| doBatchExpire(selected, commitIdentifier); | ||
| } | ||
| } | ||
| return expired; | ||
| } | ||
|
|
||
| private void doBatchExpire( | ||
| List<Map<String, String>> expiredBatchPartitions, long commitIdentifier) { | ||
| List<ExpiredPartition> expiredBatchPartitions, long commitIdentifier) { | ||
| // Drop the selected partitions themselves: a stored name is not always parsed back to the | ||
| // same partition, e.g. when it equals the default partition name or is a TIMESTAMP_LTZ | ||
| // value spelled in UTC but parsed in the local time zone. | ||
| List<BinaryRow> partitions = | ||
| expiredBatchPartitions.stream().map(p -> p.partition).collect(Collectors.toList()); | ||
| if (partitionModification != null) { | ||
| // the metastore still drops its metadata by the stored names | ||
| try { | ||
| partitionModification.dropPartitions(expiredBatchPartitions); | ||
| partitionModification.dropBinaryPartitions(partitions); | ||
| } catch (Catalog.TableNotExistException e) { | ||
| throw new RuntimeException(e); | ||
| } | ||
| } else { | ||
| commit.dropPartitions(expiredBatchPartitions, commitIdentifier); | ||
| commit.dropBinaryPartitions(partitions, commitIdentifier); | ||
| } | ||
| } | ||
|
|
||
| private List<Map<String, String>> convertToPartitionString( | ||
| List<List<String>> expiredPartValues) { | ||
| // Sort on the joined form, but build the partition from the original values: a value | ||
| // may itself contain DELIMITER, and splitting the joined string back apart would then | ||
| // shift every following field by one. | ||
| return expiredPartValues.stream() | ||
| .sorted(Comparator.comparing(values -> String.join(DELIMITER, values))) | ||
| .map(values -> strategy.toPartitionString(values.toArray())) | ||
| .limit(Math.min(expiredPartValues.size(), maxExpireNum)) | ||
| .collect(Collectors.toList()); | ||
| /** | ||
| * A partition selected for expiration. The stored name is for reporting, while deletion uses | ||
| * the original partition. | ||
| */ | ||
| private static class ExpiredPartition { | ||
|
|
||
| private final BinaryRow partition; | ||
| private final String sortKey; | ||
| private final Map<String, String> name; | ||
|
|
||
| private ExpiredPartition(BinaryRow partition, String sortKey, Map<String, String> name) { | ||
| this.partition = partition; | ||
| this.sortKey = sortKey; | ||
| this.name = name; | ||
| } | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
[P1] Preserve the LTZ partition key when dropping legacy names
The legacy LTZ name is a UTC
Timestamp.toString()spelling, butcommit.dropPartitionsreparses the supplied string as local time. Replacing the normalized value with this name therefore targets a different key. I reproduced actual writes/expiry/readback withTIMESTAMP_LTZ(0), legacy partition names andAmerica/New_York: write instants2023-01-01T00:00Zand2023-01-01T05:00Z, then expire at local January 1 22:00 with one-day retention. The returned selection correctly contains only legacy name2023-01-01T00:00(the 27-hour-old partition), but readback leaves that old row and removes the 05:00Z row, which is only 22 hours old. The legacy name was reparsed as midnight New York, i.e. 05:00Z. The new normalization activates this deletion path where the old default extractor skipped the ISO values. Please make truncation round-trip the original typed LTZ key (or use a legacy-aware conversion), keeping stored/HMS spelling separate where necessary, and cover a negative-offset timezone with two such partitions. This is a logical deletion of the wrong live partition, not a claim of immediate physical file erasure.