From 8f80852b377f6f16d2dbdf654bc500a9101f52ac Mon Sep 17 00:00:00 2001 From: sychen Date: Wed, 28 Oct 2026 20:18:49 +0800 Subject: [PATCH 1/5] [core] Expire TIMESTAMP partitions with the values-time strategy Partition expiration with the values-time strategy never expired a partition keyed by a TIMESTAMP or TIMESTAMP_LTZ column: the value was rendered with Timestamp#toString, whose ISO form (2023-01-01T00:00) the default partition time formatter cannot parse. Render such values as the SQL timestamp string for time extraction, and use the stored partition name when dropping them, as already done for DATE keys. --- .../operation/NormalPartitionExpire.java | 6 +- .../partition/PartitionExpireStrategy.java | 41 +++++++++++-- .../paimon/operation/PartitionExpireTest.java | 59 +++++++++++++++++++ 3 files changed, 99 insertions(+), 7 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/NormalPartitionExpire.java b/paimon-core/src/main/java/org/apache/paimon/operation/NormalPartitionExpire.java index 6abffab58d7e..f5c418ba2e7f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/NormalPartitionExpire.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/NormalPartitionExpire.java @@ -34,7 +34,6 @@ import javax.annotation.Nullable; import java.time.Duration; -import java.time.LocalDate; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Collection; @@ -160,14 +159,15 @@ private List> doExpire( List> expiredPartValues = new ArrayList<>(partitionEntries.size()); for (PartitionEntry partition : partitionEntries) { Object[] array = strategy.convertPartition(partition.partition()); - // DATE values are normalized for time extraction; use the stored names for dropping. + // DATE and TIMESTAMP values are normalized for time extraction; use the stored + // names for dropping. Map partitionSpec = commit.pathFactory() .partitionComputer() .generatePartValues(partition.partition()); int index = 0; for (String value : partitionSpec.values()) { - if (array[index] instanceof LocalDate) { + if (array[index] != null && strategy.isNormalizedForTimeExtraction(index)) { array[index] = value; } index++; diff --git a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionExpireStrategy.java b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionExpireStrategy.java index fc00691c9a6a..bdbe76d416c6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionExpireStrategy.java +++ b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionExpireStrategy.java @@ -22,10 +22,13 @@ import org.apache.paimon.catalog.CatalogLoader; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.Timestamp; import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.operation.FileStoreScan; -import org.apache.paimon.types.DataTypeRoot; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypeChecks; import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.DateTimeUtils; import org.apache.paimon.utils.RowDataToObjectArrayConverter; import org.apache.paimon.shade.guava30.com.google.common.primitives.UnsignedBytes; @@ -78,13 +81,43 @@ public List toPartitionValue(Object[] array) { return list; } + /** + * Whether {@link #convertPartition} renders the value at this index in a form meant for time + * extraction that can differ from the stored partition name. + */ + public boolean isNormalizedForTimeExtraction(int index) { + switch (toObjectArrayConverter.rowType().getTypeAt(index).getTypeRoot()) { + case DATE: + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return true; + default: + return false; + } + } + public Object[] convertPartition(BinaryRow partition) { Object[] values = toObjectArrayConverter.convert(partition); RowType partitionType = toObjectArrayConverter.rowType(); for (int index = 0; index < values.length; index++) { - if (values[index] != null - && partitionType.getTypeAt(index).getTypeRoot() == DataTypeRoot.DATE) { - values[index] = LocalDate.ofEpochDay((Integer) values[index]); + if (values[index] == null) { + continue; + } + DataType type = partitionType.getTypeAt(index); + switch (type.getTypeRoot()) { + case DATE: + values[index] = LocalDate.ofEpochDay((Integer) values[index]); + break; + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + // Timestamp#toString uses the ISO 'T' separator, which the default + // partition time formatter cannot parse. Render it as the SQL string. + values[index] = + DateTimeUtils.formatTimestamp( + (Timestamp) values[index], DataTypeChecks.getPrecision(type)); + break; + default: + break; } } return values; diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java index 4b07a3fbbf46..10020a0cc959 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java @@ -337,6 +337,65 @@ public void testDateAndHourPartitionValues() { assertThat(strategy.isExpired(partitionDate.atTime(13, 0), partition)).isTrue(); } + @ParameterizedTest + @CsvSource({ + "values-time, false", + "values-time, true", + "update-time, false", + "update-time, true" + }) + public void testExpireTimestampPartitionWithoutMetastore( + String expirationStrategy, boolean legacyPartitionName) throws Exception { + SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), path); + Map tableOptions = new HashMap<>(); + tableOptions.put(METASTORE_PARTITIONED_TABLE.key(), "false"); + tableOptions.put( + CoreOptions.PARTITION_GENERATE_LEGACY_NAME.key(), + String.valueOf(legacyPartitionName)); + schemaManager.createTable( + new Schema( + RowType.of(DataTypes.TIMESTAMP(0), VarCharType.STRING_TYPE).getFields(), + singletonList("f0"), + emptyList(), + tableOptions, + "")); + newTable(); + org.apache.paimon.data.Timestamp expiredTs = + org.apache.paimon.data.Timestamp.fromLocalDateTime(date(1)); + org.apache.paimon.data.Timestamp retainedTs = + org.apache.paimon.data.Timestamp.fromLocalDateTime(date(5)); + write(GenericRow.of(expiredTs, BinaryString.fromString("expired"))); + write(GenericRow.of(retainedTs, BinaryString.fromString("retained"))); + + Map options = new HashMap<>(); + options.put(PARTITION_EXPIRATION_TIME.key(), "2 d"); + options.put(CoreOptions.PARTITION_EXPIRATION_STRATEGY.key(), expirationStrategy); + options.put(CoreOptions.PARTITION_TIMESTAMP_PATTERN.key(), "$f0"); + table = table.copy(options); + NormalPartitionExpire expire = + (NormalPartitionExpire) table.store().newPartitionExpire("", table); + expire.setLastCheck(date(1)); + // the legacy name is Timestamp#toString, the new name is the SQL timestamp string + String expiredName = legacyPartitionName ? "2023-01-01T00:00" : "2023-01-01 00:00:00"; + String retainedName = legacyPartitionName ? "2023-01-05T00:00" : "2023-01-05 00:00:00"; + List> expectedExpired = new ArrayList<>(); + expectedExpired.add(Collections.singletonMap("f0", expiredName)); + LocalDateTime checkTime = date(6); + List expectedRemaining = singletonList(retainedTs); + if ("update-time".equals(expirationStrategy)) { + checkTime = LocalDateTime.now().plusDays(3); + expectedExpired.add(Collections.singletonMap("f0", retainedName)); + expectedRemaining = emptyList(); + } + assertThat(expire.expire(checkTime, Long.MAX_VALUE)) + .containsExactlyElementsOf(expectedExpired); + List remaining = new ArrayList<>(); + table.newRead() + .createReader(table.newScan().plan().splits()) + .forEachRemaining(row -> remaining.add(row.getTimestamp(0, 0))); + assertThat(remaining).containsExactlyElementsOf(expectedRemaining); + } + @ParameterizedTest @ValueSource(strings = {"values-time", "update-time"}) public void testExpireDatePartitionWithoutMetastore(String expirationStrategy) From 39bc84f25432bf80defc951ffff37972d4d7d3ca Mon Sep 17 00:00:00 2001 From: sychen Date: Mon, 2 Nov 2026 21:08:50 +0800 Subject: [PATCH 2/5] [core] Respect LTZ time zone and custom formatters when expiring TIMESTAMP partitions --- .../partition/PartitionExpireStrategy.java | 33 ++++++- .../PartitionValuesTimeExpireStrategy.java | 2 +- .../paimon/operation/PartitionExpireTest.java | 97 +++++++++++++++++-- 3 files changed, 121 insertions(+), 11 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionExpireStrategy.java b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionExpireStrategy.java index bdbe76d416c6..f09466e9f17d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionExpireStrategy.java +++ b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionExpireStrategy.java @@ -27,6 +27,7 @@ import org.apache.paimon.operation.FileStoreScan; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypeChecks; +import org.apache.paimon.types.DataTypeRoot; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.DateTimeUtils; import org.apache.paimon.utils.RowDataToObjectArrayConverter; @@ -45,6 +46,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.TimeZone; /** Strategy for partition expiration. */ public abstract class PartitionExpireStrategy implements Serializable { @@ -54,11 +56,20 @@ public abstract class PartitionExpireStrategy implements Serializable { protected final List partitionKeys; protected final String partitionDefaultName; private final RowDataToObjectArrayConverter toObjectArrayConverter; + @Nullable private final String timestampFormatter; public PartitionExpireStrategy(RowType partitionType, String partitionDefaultName) { + this(partitionType, partitionDefaultName, null); + } + + public PartitionExpireStrategy( + RowType partitionType, + String partitionDefaultName, + @Nullable String timestampFormatter) { this.toObjectArrayConverter = new RowDataToObjectArrayConverter(partitionType); this.partitionKeys = partitionType.getFieldNames(); this.partitionDefaultName = partitionDefaultName; + this.timestampFormatter = timestampFormatter; } public Map toPartitionString(Object[] array) { @@ -110,11 +121,8 @@ public Object[] convertPartition(BinaryRow partition) { break; case TIMESTAMP_WITHOUT_TIME_ZONE: case TIMESTAMP_WITH_LOCAL_TIME_ZONE: - // Timestamp#toString uses the ISO 'T' separator, which the default - // partition time formatter cannot parse. Render it as the SQL string. values[index] = - DateTimeUtils.formatTimestamp( - (Timestamp) values[index], DataTypeChecks.getPrecision(type)); + formatTimestampForTimeExtraction((Timestamp) values[index], type); break; default: break; @@ -123,6 +131,23 @@ public Object[] convertPartition(BinaryRow partition) { return values; } + private String formatTimestampForTimeExtraction(Timestamp timestamp, DataType type) { + if (type.is(DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE)) { + // Use the local wall-clock time, consistent with the current partition names and + // with the local current time used for expiration. + timestamp = + DateTimeUtils.timestampWithLocalZoneToTimestamp( + timestamp, TimeZone.getDefault()); + } + if (timestampFormatter != null) { + // Keep the Timestamp#toString input an explicitly configured formatter expects. + return timestamp.toString(); + } + // Timestamp#toString uses the ISO 'T' separator, which the default partition time + // formatter cannot parse. Render it as the SQL timestamp string. + return DateTimeUtils.formatTimestamp(timestamp, DataTypeChecks.getPrecision(type)); + } + public abstract List selectExpiredPartitions( FileStoreScan scan, LocalDateTime expirationTime); diff --git a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionValuesTimeExpireStrategy.java b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionValuesTimeExpireStrategy.java index 0291bcc6c916..35a2b3a42f9e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionValuesTimeExpireStrategy.java +++ b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionValuesTimeExpireStrategy.java @@ -55,7 +55,7 @@ public class PartitionValuesTimeExpireStrategy extends PartitionExpireStrategy { private final boolean retainLatest; public PartitionValuesTimeExpireStrategy(CoreOptions options, RowType partitionType) { - super(partitionType, options.partitionDefaultName()); + super(partitionType, options.partitionDefaultName(), options.partitionTimestampFormatter()); String timePattern = options.partitionTimestampPattern(); String timeFormatter = options.partitionTimestampFormatter(); this.timeExtractor = new PartitionTimeExtractor(timePattern, timeFormatter); diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java index 10020a0cc959..a2fcb70cc83a 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java @@ -24,6 +24,7 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.Timestamp; import org.apache.paimon.data.serializer.InternalRowSerializer; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; @@ -67,6 +68,8 @@ import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; +import javax.annotation.Nullable; + import java.io.IOException; import java.time.LocalDate; import java.time.LocalDateTime; @@ -83,6 +86,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TimeZone; import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; @@ -360,10 +364,8 @@ public void testExpireTimestampPartitionWithoutMetastore( tableOptions, "")); newTable(); - org.apache.paimon.data.Timestamp expiredTs = - org.apache.paimon.data.Timestamp.fromLocalDateTime(date(1)); - org.apache.paimon.data.Timestamp retainedTs = - org.apache.paimon.data.Timestamp.fromLocalDateTime(date(5)); + Timestamp expiredTs = Timestamp.fromLocalDateTime(date(1)); + Timestamp retainedTs = Timestamp.fromLocalDateTime(date(5)); write(GenericRow.of(expiredTs, BinaryString.fromString("expired"))); write(GenericRow.of(retainedTs, BinaryString.fromString("retained"))); @@ -381,7 +383,7 @@ public void testExpireTimestampPartitionWithoutMetastore( List> expectedExpired = new ArrayList<>(); expectedExpired.add(Collections.singletonMap("f0", expiredName)); LocalDateTime checkTime = date(6); - List expectedRemaining = singletonList(retainedTs); + List expectedRemaining = singletonList(retainedTs); if ("update-time".equals(expirationStrategy)) { checkTime = LocalDateTime.now().plusDays(3); expectedExpired.add(Collections.singletonMap("f0", retainedName)); @@ -389,13 +391,96 @@ public void testExpireTimestampPartitionWithoutMetastore( } assertThat(expire.expire(checkTime, Long.MAX_VALUE)) .containsExactlyElementsOf(expectedExpired); - List remaining = new ArrayList<>(); + List remaining = new ArrayList<>(); table.newRead() .createReader(table.newScan().plan().splits()) .forEachRemaining(row -> remaining.add(row.getTimestamp(0, 0))); assertThat(remaining).containsExactlyElementsOf(expectedRemaining); } + @Test + public void testExpireLocalZonedTimestampPartitionInNonUtcZone() throws Exception { + TimeZone defaultZone = TimeZone.getDefault(); + TimeZone.setDefault(TimeZone.getTimeZone("Asia/Shanghai")); + try { + // current partition names render LTZ values in the default time zone + createTimestampPartitionedTable(DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(0), false); + // instant 2023-01-01T00:00Z is local 2023-01-01 08:00:00 + Timestamp expiredTs = Timestamp.fromLocalDateTime(date(1)); + // instant 2023-01-01T20:00Z is local 2023-01-02 04:00:00 + Timestamp retainedTs = Timestamp.fromLocalDateTime(date(1).plusHours(20)); + write(GenericRow.of(expiredTs, BinaryString.fromString("expired"))); + write(GenericRow.of(retainedTs, BinaryString.fromString("retained"))); + + NormalPartitionExpire expire = newValuesTimeExpire("1 d", null); + // local 2023-01-03 02:00:00: the retained partition is only 22 hours old + assertThat(expire.expire(date(3).plusHours(2), Long.MAX_VALUE)) + .containsExactly(Collections.singletonMap("f0", "2023-01-01 08:00:00")); + assertThat(readTimestamps()).containsExactly(retainedTs); + } finally { + TimeZone.setDefault(defaultZone); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testExpireTimestampPartitionWithCustomFormatter(boolean legacyPartitionName) + throws Exception { + createTimestampPartitionedTable(DataTypes.TIMESTAMP(0), legacyPartitionName); + Timestamp expiredTs = Timestamp.fromLocalDateTime(date(1)); + Timestamp retainedTs = Timestamp.fromLocalDateTime(date(5)); + write(GenericRow.of(expiredTs, BinaryString.fromString("expired"))); + write(GenericRow.of(retainedTs, BinaryString.fromString("retained"))); + + // the formatter matches the Timestamp#toString input it has always received + NormalPartitionExpire expire = newValuesTimeExpire("2 d", "yyyy-MM-dd'T'HH:mm"); + String expiredName = legacyPartitionName ? "2023-01-01T00:00" : "2023-01-01 00:00:00"; + assertThat(expire.expire(date(6), Long.MAX_VALUE)) + .containsExactly(Collections.singletonMap("f0", expiredName)); + assertThat(readTimestamps()).containsExactly(retainedTs); + } + + private void createTimestampPartitionedTable(DataType partitionType, boolean legacyName) + throws Exception { + SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), path); + Map tableOptions = new HashMap<>(); + tableOptions.put(METASTORE_PARTITIONED_TABLE.key(), "false"); + tableOptions.put( + CoreOptions.PARTITION_GENERATE_LEGACY_NAME.key(), String.valueOf(legacyName)); + schemaManager.createTable( + new Schema( + RowType.of(partitionType, VarCharType.STRING_TYPE).getFields(), + singletonList("f0"), + emptyList(), + tableOptions, + "")); + newTable(); + } + + private NormalPartitionExpire newValuesTimeExpire( + String expirationTime, @Nullable String formatter) { + Map options = new HashMap<>(); + options.put(PARTITION_EXPIRATION_TIME.key(), expirationTime); + options.put(CoreOptions.PARTITION_EXPIRATION_STRATEGY.key(), "values-time"); + options.put(CoreOptions.PARTITION_TIMESTAMP_PATTERN.key(), "$f0"); + if (formatter != null) { + options.put(CoreOptions.PARTITION_TIMESTAMP_FORMATTER.key(), formatter); + } + table = table.copy(options); + NormalPartitionExpire expire = + (NormalPartitionExpire) table.store().newPartitionExpire("", table); + expire.setLastCheck(date(1)); + return expire; + } + + private List readTimestamps() throws Exception { + List timestamps = new ArrayList<>(); + table.newRead() + .createReader(table.newScan().plan().splits()) + .forEachRemaining(row -> timestamps.add(row.getTimestamp(0, 0))); + return timestamps; + } + @ParameterizedTest @ValueSource(strings = {"values-time", "update-time"}) public void testExpireDatePartitionWithoutMetastore(String expirationStrategy) From d3518feb2c44a23f66d0db40ec4ea74459f7b3ed Mon Sep 17 00:00:00 2001 From: sychen Date: Wed, 4 Nov 2026 16:16:19 +0800 Subject: [PATCH 3/5] [core] Limit values-time TIMESTAMP partition expiration to TIMESTAMP without time zone A legacy TIMESTAMP_LTZ partition name is a UTC Timestamp#toString spelling, while partition specs passed to dropPartitions are parsed in the default time zone, so expiring such partitions outside UTC could drop a different, live partition. Reliable LTZ expiration needs dropping by the stored partition row rather than by a re-parsed string, which is out of scope here. Normalize only TIMESTAMP without time zone values, keep TIMESTAMP_LTZ on its previous behavior, and cover that LTZ partitions are neither expired nor dropped by default in a non-UTC zone. Co-Authored-By: Claude Opus 5.5 --- .../partition/PartitionExpireStrategy.java | 11 ------- .../paimon/operation/PartitionExpireTest.java | 30 +++++++++---------- 2 files changed, 15 insertions(+), 26 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionExpireStrategy.java b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionExpireStrategy.java index f09466e9f17d..bfc6e32f09c9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionExpireStrategy.java +++ b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionExpireStrategy.java @@ -27,7 +27,6 @@ import org.apache.paimon.operation.FileStoreScan; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypeChecks; -import org.apache.paimon.types.DataTypeRoot; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.DateTimeUtils; import org.apache.paimon.utils.RowDataToObjectArrayConverter; @@ -46,7 +45,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.TimeZone; /** Strategy for partition expiration. */ public abstract class PartitionExpireStrategy implements Serializable { @@ -100,7 +98,6 @@ public boolean isNormalizedForTimeExtraction(int index) { switch (toObjectArrayConverter.rowType().getTypeAt(index).getTypeRoot()) { case DATE: case TIMESTAMP_WITHOUT_TIME_ZONE: - case TIMESTAMP_WITH_LOCAL_TIME_ZONE: return true; default: return false; @@ -120,7 +117,6 @@ public Object[] convertPartition(BinaryRow partition) { values[index] = LocalDate.ofEpochDay((Integer) values[index]); break; case TIMESTAMP_WITHOUT_TIME_ZONE: - case TIMESTAMP_WITH_LOCAL_TIME_ZONE: values[index] = formatTimestampForTimeExtraction((Timestamp) values[index], type); break; @@ -132,13 +128,6 @@ public Object[] convertPartition(BinaryRow partition) { } private String formatTimestampForTimeExtraction(Timestamp timestamp, DataType type) { - if (type.is(DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE)) { - // Use the local wall-clock time, consistent with the current partition names and - // with the local current time used for expiration. - timestamp = - DateTimeUtils.timestampWithLocalZoneToTimestamp( - timestamp, TimeZone.getDefault()); - } if (timestampFormatter != null) { // Keep the Timestamp#toString input an explicitly configured formatter expects. return timestamp.toString(); diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java index a2fcb70cc83a..234b2d276197 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java @@ -398,25 +398,25 @@ public void testExpireTimestampPartitionWithoutMetastore( assertThat(remaining).containsExactlyElementsOf(expectedRemaining); } - @Test - public void testExpireLocalZonedTimestampPartitionInNonUtcZone() throws Exception { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testLocalZonedTimestampPartitionIsNotExpiredByDefault(boolean legacyPartitionName) + throws Exception { TimeZone defaultZone = TimeZone.getDefault(); - TimeZone.setDefault(TimeZone.getTimeZone("Asia/Shanghai")); + TimeZone.setDefault(TimeZone.getTimeZone("America/New_York")); try { - // current partition names render LTZ values in the default time zone - createTimestampPartitionedTable(DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(0), false); - // instant 2023-01-01T00:00Z is local 2023-01-01 08:00:00 - Timestamp expiredTs = Timestamp.fromLocalDateTime(date(1)); - // instant 2023-01-01T20:00Z is local 2023-01-02 04:00:00 - Timestamp retainedTs = Timestamp.fromLocalDateTime(date(1).plusHours(20)); - write(GenericRow.of(expiredTs, BinaryString.fromString("expired"))); - write(GenericRow.of(retainedTs, BinaryString.fromString("retained"))); + // TIMESTAMP_LTZ values are not normalized for time extraction: a legacy name is a UTC + // spelling that a partition spec would parse back in the default time zone + createTimestampPartitionedTable( + DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(0), legacyPartitionName); + Timestamp oldTs = Timestamp.fromLocalDateTime(date(1)); + Timestamp freshTs = Timestamp.fromLocalDateTime(date(1).plusHours(5)); + write(GenericRow.of(oldTs, BinaryString.fromString("old"))); + write(GenericRow.of(freshTs, BinaryString.fromString("fresh"))); NormalPartitionExpire expire = newValuesTimeExpire("1 d", null); - // local 2023-01-03 02:00:00: the retained partition is only 22 hours old - assertThat(expire.expire(date(3).plusHours(2), Long.MAX_VALUE)) - .containsExactly(Collections.singletonMap("f0", "2023-01-01 08:00:00")); - assertThat(readTimestamps()).containsExactly(retainedTs); + assertThat(expire.expire(date(1).plusHours(22), Long.MAX_VALUE)).isEmpty(); + assertThat(readTimestamps()).containsExactlyInAnyOrder(oldTs, freshTs); } finally { TimeZone.setDefault(defaultZone); } From b79af8fcb9ba946aa127d7e63b28528c4aa16c52 Mon Sep 17 00:00:00 2001 From: sychen Date: Fri, 6 Nov 2026 18:01:48 +0800 Subject: [PATCH 4/5] dropBinaryPartitions --- .../paimon/operation/FileStoreCommit.java | 11 ++ .../paimon/operation/FileStoreCommitImpl.java | 12 +++ .../operation/NormalPartitionExpire.java | 75 ++++++++----- .../paimon/operation/PartitionExpireTest.java | 102 +++++++++++++++--- 4 files changed, 164 insertions(+), 36 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommit.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommit.java index 050a345a0e3b..b778d2285f2f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommit.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommit.java @@ -74,6 +74,17 @@ public interface FileStoreCommit extends AutoCloseable { */ void dropPartitions(List> partitions, long commitIdentifier); + /** + * Drop multiple partitions given by their exact partition values. Unlike {@link + * #dropPartitions}, the partitions are not parsed back from their names, so a name that is + * ambiguous (e.g. equal to the default partition name) or time-zone dependent still drops + * exactly the given partition. The {@link Snapshot.CommitKind} of generated snapshot is {@link + * Snapshot.CommitKind#OVERWRITE}. + * + * @param partitions A list of partitions. NOTE: cannot be empty! + */ + void dropBinaryPartitions(List partitions, long commitIdentifier); + void truncateTable(long commitIdentifier); /** Compact the manifest entries only. */ diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java index 860835de53da..5e7b251212e3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java @@ -716,6 +716,18 @@ public void dropPartitions(List> partitions, long commitIden partitionFilter, emptyList(), emptyList(), commitIdentifier, null, new HashMap<>()); } + @Override + public void dropBinaryPartitions(List partitions, long commitIdentifier) { + checkArgument(!partitions.isEmpty(), "Partitions list cannot be empty."); + tryOverwritePartition( + PartitionPredicate.fromMultiple(partitionType, partitions), + emptyList(), + emptyList(), + commitIdentifier, + null, + new HashMap<>()); + } + @Override public void truncateTable(long commitIdentifier) { tryOverwritePartition( diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/NormalPartitionExpire.java b/paimon-core/src/main/java/org/apache/paimon/operation/NormalPartitionExpire.java index f5c418ba2e7f..dc064b61cf68 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/NormalPartitionExpire.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/NormalPartitionExpire.java @@ -156,11 +156,11 @@ private List> doExpire( LocalDateTime expireDateTime, long commitIdentifier) { List partitionEntries = strategy.selectExpiredPartitions(scan, expireDateTime); - List> expiredPartValues = new ArrayList<>(partitionEntries.size()); + List expiredPartitions = new ArrayList<>(partitionEntries.size()); for (PartitionEntry partition : partitionEntries) { Object[] array = strategy.convertPartition(partition.partition()); - // DATE and TIMESTAMP values are normalized for time extraction; use the stored - // names for dropping. + // DATE and TIMESTAMP values are normalized for time extraction; report the stored + // names instead. Map partitionSpec = commit.pathFactory() .partitionComputer() @@ -172,48 +172,75 @@ private List> doExpire( } index++; } - expiredPartValues.add(strategy.toPartitionValue(array)); + List values = strategy.toPartitionValue(array); + // Sort on the joined form, but build the partition from the original values: a + // value may itself contain DELIMITER, and splitting the joined string back apart + // would then shift every following field by one. + expiredPartitions.add( + new ExpiredPartition( + partition.partition(), + String.join(DELIMITER, values), + strategy.toPartitionString(values.toArray()))); } - List> expired = new ArrayList<>(); - if (!expiredPartValues.isEmpty()) { - // convert partition value to partition string, and limit the partition num - expired = convertToPartitionString(expiredPartValues); + List selected = + expiredPartitions.stream() + .sorted(Comparator.comparing(p -> p.sortKey)) + .limit(Math.min(expiredPartitions.size(), maxExpireNum)) + .collect(Collectors.toList()); + List> expired = + selected.stream().map(p -> p.name).collect(Collectors.toList()); + if (!selected.isEmpty()) { LOG.info("Expire Partitions: {}", expired); - if (expireBatchSize > 0 && expireBatchSize < expired.size()) { - Lists.partition(expired, expireBatchSize) + if (expireBatchSize > 0 && expireBatchSize < selected.size()) { + Lists.partition(selected, expireBatchSize) .forEach( expiredBatchPartitions -> doBatchExpire(expiredBatchPartitions, commitIdentifier)); } else { - doBatchExpire(expired, commitIdentifier); + doBatchExpire(selected, commitIdentifier); } } return expired; } private void doBatchExpire( - List> expiredBatchPartitions, long commitIdentifier) { + List expiredBatchPartitions, long commitIdentifier) { if (partitionModification != null) { + // the metastore identifies partitions by their stored names + List> names = + expiredBatchPartitions.stream().map(p -> p.name).collect(Collectors.toList()); try { - partitionModification.dropPartitions(expiredBatchPartitions); + partitionModification.dropPartitions(names); } catch (Catalog.TableNotExistException e) { throw new RuntimeException(e); } } else { - commit.dropPartitions(expiredBatchPartitions, commitIdentifier); + // Drop the selected partitions themselves: a stored name is not always parsed back + // to the same partition, e.g. when it equals the default partition name or is a + // TIMESTAMP_LTZ value spelled in UTC but parsed in the local time zone. + commit.dropBinaryPartitions( + expiredBatchPartitions.stream() + .map(p -> p.partition) + .collect(Collectors.toList()), + commitIdentifier); } } - private List> convertToPartitionString( - List> expiredPartValues) { - // Sort on the joined form, but build the partition from the original values: a value - // may itself contain DELIMITER, and splitting the joined string back apart would then - // shift every following field by one. - return expiredPartValues.stream() - .sorted(Comparator.comparing(values -> String.join(DELIMITER, values))) - .map(values -> strategy.toPartitionString(values.toArray())) - .limit(Math.min(expiredPartValues.size(), maxExpireNum)) - .collect(Collectors.toList()); + /** + * A partition selected for expiration. The stored name is for reporting and the metastore, + * while local deletion uses the original partition. + */ + private static class ExpiredPartition { + + private final BinaryRow partition; + private final String sortKey; + private final Map name; + + private ExpiredPartition(BinaryRow partition, String sortKey, Map name) { + this.partition = partition; + this.sortKey = sortKey; + this.name = name; + } } } diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java index 234b2d276197..4ed982cb33bb 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java @@ -405,8 +405,8 @@ public void testLocalZonedTimestampPartitionIsNotExpiredByDefault(boolean legacy TimeZone defaultZone = TimeZone.getDefault(); TimeZone.setDefault(TimeZone.getTimeZone("America/New_York")); try { - // TIMESTAMP_LTZ values are not normalized for time extraction: a legacy name is a UTC - // spelling that a partition spec would parse back in the default time zone + // TIMESTAMP_LTZ values are not normalized for time extraction, so their ISO spelling + // is still not parsed by the default extractor and they are not expired createTimestampPartitionedTable( DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(0), legacyPartitionName); Timestamp oldTs = Timestamp.fromLocalDateTime(date(1)); @@ -440,26 +440,104 @@ public void testExpireTimestampPartitionWithCustomFormatter(boolean legacyPartit assertThat(readTimestamps()).containsExactly(retainedTs); } + @Test + public void testExpireTimestampPartitionNamedAsDefaultPartition() throws Exception { + // The epoch partition is stored under the same name as the NULL partition, so parsing + // the name back would drop the NULL partition instead of the selected one. + Map tableOptions = new HashMap<>(); + tableOptions.put(CoreOptions.PARTITION_DEFAULT_NAME.key(), "1970-01-01 00:00:00"); + tableOptions.put(CoreOptions.PARTITION_GENERATE_LEGACY_NAME.key(), "false"); + createPartitionedTable( + RowType.of(DataTypes.TIMESTAMP(0), VarCharType.STRING_TYPE), + singletonList("f0"), + tableOptions); + write(GenericRow.of(Timestamp.fromEpochMillis(0), BinaryString.fromString("epoch"))); + write(GenericRow.of(null, BinaryString.fromString("null"))); + + NormalPartitionExpire expire = + newValuesTimeExpire("2 d", "yyyy-MM-dd'T'HH:mm", Collections.emptyMap()); + assertThat(expire.expire(date(6), Long.MAX_VALUE)) + .containsExactly(Collections.singletonMap("f0", "1970-01-01 00:00:00")); + assertThat(readStrings(1)).containsExactly("null"); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testExpireDropsSelectedLocalZonedTimestampPartition(boolean legacyPartitionName) + throws Exception { + TimeZone defaultZone = TimeZone.getDefault(); + TimeZone.setDefault(TimeZone.getTimeZone("America/New_York")); + try { + // The LTZ key is not used for time extraction. Its reported value is the UTC + // Timestamp#toString spelling, which a partition spec would parse back in the default + // time zone and match another partition. + createPartitionedTable( + RowType.of( + DataTypes.TIMESTAMP(0), + DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(0), + VarCharType.STRING_TYPE), + Arrays.asList("f0", "f1"), + Collections.singletonMap( + CoreOptions.PARTITION_GENERATE_LEGACY_NAME.key(), + String.valueOf(legacyPartitionName))); + Timestamp day = Timestamp.fromLocalDateTime(date(1)); + write(GenericRow.of(day, day, BinaryString.fromString("00:00Z"))); + write( + GenericRow.of( + day, + Timestamp.fromLocalDateTime(date(1).plusHours(5)), + BinaryString.fromString("05:00Z"))); + + NormalPartitionExpire expire = + newValuesTimeExpire( + "2 d", + null, + Collections.singletonMap( + CoreOptions.PARTITION_EXPIRATION_MAX_NUM.key(), "1")); + assertThat(expire.expire(date(6), Long.MAX_VALUE)).hasSize(1); + assertThat(readStrings(2)).containsExactly("05:00Z"); + } finally { + TimeZone.setDefault(defaultZone); + } + } + private void createTimestampPartitionedTable(DataType partitionType, boolean legacyName) throws Exception { + createPartitionedTable( + RowType.of(partitionType, VarCharType.STRING_TYPE), + singletonList("f0"), + Collections.singletonMap( + CoreOptions.PARTITION_GENERATE_LEGACY_NAME.key(), + String.valueOf(legacyName))); + } + + private void createPartitionedTable( + RowType rowType, List partitionKeys, Map extraOptions) + throws Exception { SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), path); - Map tableOptions = new HashMap<>(); + Map tableOptions = new HashMap<>(extraOptions); tableOptions.put(METASTORE_PARTITIONED_TABLE.key(), "false"); - tableOptions.put( - CoreOptions.PARTITION_GENERATE_LEGACY_NAME.key(), String.valueOf(legacyName)); schemaManager.createTable( - new Schema( - RowType.of(partitionType, VarCharType.STRING_TYPE).getFields(), - singletonList("f0"), - emptyList(), - tableOptions, - "")); + new Schema(rowType.getFields(), partitionKeys, emptyList(), tableOptions, "")); newTable(); } + private List readStrings(int index) throws Exception { + List values = new ArrayList<>(); + table.newRead() + .createReader(table.newScan().plan().splits()) + .forEachRemaining(row -> values.add(row.getString(index).toString())); + return values; + } + private NormalPartitionExpire newValuesTimeExpire( String expirationTime, @Nullable String formatter) { - Map options = new HashMap<>(); + return newValuesTimeExpire(expirationTime, formatter, Collections.emptyMap()); + } + + private NormalPartitionExpire newValuesTimeExpire( + String expirationTime, @Nullable String formatter, Map extraOptions) { + Map options = new HashMap<>(extraOptions); options.put(PARTITION_EXPIRATION_TIME.key(), expirationTime); options.put(CoreOptions.PARTITION_EXPIRATION_STRATEGY.key(), "values-time"); options.put(CoreOptions.PARTITION_TIMESTAMP_PATTERN.key(), "$f0"); From ed262578d9bc2ac3f7e52941c5bc9e5a65ae6727 Mon Sep 17 00:00:00 2001 From: sychen Date: Tue, 10 Nov 2026 12:58:13 +0800 Subject: [PATCH 5/5] hms --- .../apache/paimon/catalog/CachingCatalog.java | 10 ++ .../org/apache/paimon/catalog/Catalog.java | 34 +++++ .../paimon/catalog/DelegateCatalog.java | 7 ++ .../operation/NormalPartitionExpire.java | 24 ++-- .../paimon/table/PartitionModification.java | 14 +++ .../paimon/table/sink/TableCommitImpl.java | 9 ++ .../paimon/operation/PartitionExpireTest.java | 57 +++++++-- .../org/apache/paimon/hive/HiveCatalog.java | 45 ++++++- .../apache/paimon/hive/HiveCatalogTest.java | 119 ++++++++++++++++++ 9 files changed, 294 insertions(+), 25 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java index 037f7ea2bfba..81f2c8c714d4 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java @@ -20,6 +20,7 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.annotation.VisibleForTesting; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.fs.Path; import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; @@ -390,6 +391,15 @@ public void dropPartitions(Identifier identifier, List> part } } + @Override + public void dropBinaryPartitions(Identifier identifier, List partitions) + throws TableNotExistException { + wrapped.dropBinaryPartitions(identifier, partitions); + if (partitionCache != null) { + partitionCache.invalidate(identifier); + } + } + @Override public void alterPartitions(Identifier identifier, List partitions) throws TableNotExistException { diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java index 88390954d1ac..8fd4827a9b31 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java @@ -23,6 +23,7 @@ import org.apache.paimon.annotation.Public; import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.consumer.ConsumerInfo; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.function.Function; import org.apache.paimon.function.FunctionChange; import org.apache.paimon.operation.Lock; @@ -39,6 +40,7 @@ import org.apache.paimon.table.Table; import org.apache.paimon.table.TableSnapshot; import org.apache.paimon.table.sink.BatchTableCommit; +import org.apache.paimon.table.sink.TableCommitImpl; import org.apache.paimon.utils.SnapshotNotExistException; import org.apache.paimon.view.View; import org.apache.paimon.view.ViewChange; @@ -1155,6 +1157,38 @@ default void dropPartitions(Identifier identifier, List> par } } + /** + * Drop partitions of the specify table given by their partition rows. Ignore non-existent + * partitions. + * + *

Unlike {@link #dropPartitions(Identifier, List)}, the data is dropped by the exact + * partition rows rather than by partition specs parsed back from strings. A partition name is + * not always parsed back to the same partition, e.g. when it equals the default partition name + * or is a TIMESTAMP_LTZ value. Partition metadata kept by name (e.g. in a metastore) is still + * dropped by the stored partition names. + * + * @param identifier path of the table to drop partitions + * @param partitions partition rows to be deleted, in the partition type of the table + * @throws TableNotExistException if the table does not exist + */ + default void dropBinaryPartitions(Identifier identifier, List partitions) + throws TableNotExistException { + Table table = getTable(identifier); + try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) { + if (!(commit instanceof TableCommitImpl)) { + throw new UnsupportedOperationException( + String.format( + "Table %s does not support dropping partitions by partition rows.", + identifier)); + } + ((TableCommitImpl) commit).truncateBinaryPartitions(partitions); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + /** * Alter partitions of the specify table. For non-existent partitions, partitions will be * created directly. diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java index 8b0aa817d430..cd9fc2596f37 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java @@ -20,6 +20,7 @@ import org.apache.paimon.PagedList; import org.apache.paimon.Snapshot; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.function.Function; import org.apache.paimon.function.FunctionChange; import org.apache.paimon.operation.Lock; @@ -378,6 +379,12 @@ public void dropPartitions(Identifier identifier, List> part wrapped.dropPartitions(identifier, partitions); } + @Override + public void dropBinaryPartitions(Identifier identifier, List partitions) + throws TableNotExistException { + wrapped.dropBinaryPartitions(identifier, partitions); + } + @Override public void alterPartitions(Identifier identifier, List partitions) throws TableNotExistException { diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/NormalPartitionExpire.java b/paimon-core/src/main/java/org/apache/paimon/operation/NormalPartitionExpire.java index dc064b61cf68..e88f93a4201b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/NormalPartitionExpire.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/NormalPartitionExpire.java @@ -206,30 +206,26 @@ private List> doExpire( private void doBatchExpire( List expiredBatchPartitions, long commitIdentifier) { + // Drop the selected partitions themselves: a stored name is not always parsed back to the + // same partition, e.g. when it equals the default partition name or is a TIMESTAMP_LTZ + // value spelled in UTC but parsed in the local time zone. + List partitions = + expiredBatchPartitions.stream().map(p -> p.partition).collect(Collectors.toList()); if (partitionModification != null) { - // the metastore identifies partitions by their stored names - List> names = - expiredBatchPartitions.stream().map(p -> p.name).collect(Collectors.toList()); + // the metastore still drops its metadata by the stored names try { - partitionModification.dropPartitions(names); + partitionModification.dropBinaryPartitions(partitions); } catch (Catalog.TableNotExistException e) { throw new RuntimeException(e); } } else { - // Drop the selected partitions themselves: a stored name is not always parsed back - // to the same partition, e.g. when it equals the default partition name or is a - // TIMESTAMP_LTZ value spelled in UTC but parsed in the local time zone. - commit.dropBinaryPartitions( - expiredBatchPartitions.stream() - .map(p -> p.partition) - .collect(Collectors.toList()), - commitIdentifier); + commit.dropBinaryPartitions(partitions, commitIdentifier); } } /** - * A partition selected for expiration. The stored name is for reporting and the metastore, - * while local deletion uses the original partition. + * A partition selected for expiration. The stored name is for reporting, while deletion uses + * the original partition. */ private static class ExpiredPartition { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/PartitionModification.java b/paimon-core/src/main/java/org/apache/paimon/table/PartitionModification.java index b3850bf9bef7..bad0ab74376e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/PartitionModification.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/PartitionModification.java @@ -20,6 +20,7 @@ import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.partition.PartitionStatistics; import java.util.List; @@ -33,6 +34,13 @@ void createPartitions(List> partitions) void dropPartitions(List> partitions) throws Catalog.TableNotExistException; + /** Drop partitions given by their partition rows, see {@link Catalog#dropBinaryPartitions}. */ + default void dropBinaryPartitions(List partitions) + throws Catalog.TableNotExistException { + throw new UnsupportedOperationException( + getClass().getName() + " does not support dropping partitions by partition rows."); + } + void alterPartitions(List partitions) throws Catalog.TableNotExistException; @@ -51,6 +59,12 @@ public void dropPartitions(List> partitions) catalog.dropPartitions(identifier, partitions); } + @Override + public void dropBinaryPartitions(List partitions) + throws Catalog.TableNotExistException { + catalog.dropBinaryPartitions(identifier, partitions); + } + @Override public void alterPartitions(List partitions) throws Catalog.TableNotExistException { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java index 23f1132fd795..832e0c01f594 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java @@ -218,6 +218,15 @@ public void truncatePartitions(List> partitionSpecs) { commit.dropPartitions(partitionSpecs, COMMIT_IDENTIFIER); } + /** + * Truncate the given partitions. Unlike {@link #truncatePartitions}, the partitions are not + * parsed back from their names, see {@link FileStoreCommit#dropBinaryPartitions}. + */ + public void truncateBinaryPartitions(List partitions) { + commit.withOperation(Snapshot.Operation.TRUNCATE); + commit.dropBinaryPartitions(partitions, COMMIT_IDENTIFIER); + } + @Override public void updateStatistics(Statistics statistics) { commit.commitStatistics(statistics, COMMIT_IDENTIFIER); diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java index 4ed982cb33bb..0028aedfafc0 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java @@ -89,6 +89,7 @@ import java.util.TimeZone; import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; +import java.util.function.Consumer; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; @@ -143,6 +144,35 @@ public void createPartitions(List> partitions) @Override public void dropPartitions(List> partitions) throws Catalog.TableNotExistException { + dropPartitions( + partitions, + commit -> + commit.dropPartitions( + partitions, BatchWriteBuilder.COMMIT_IDENTIFIER)); + } + + @Override + public void dropBinaryPartitions(List partitions) + throws Catalog.TableNotExistException { + // Model HiveCatalog: metadata by the stored names, data by the rows. + List> partitionSpecs = new ArrayList<>(); + for (BinaryRow partition : partitions) { + partitionSpecs.add( + table.store() + .pathFactory() + .partitionComputer() + .generatePartValues(partition)); + } + dropPartitions( + partitionSpecs, + commit -> + commit.dropBinaryPartitions( + partitions, BatchWriteBuilder.COMMIT_IDENTIFIER)); + } + + private void dropPartitions( + List> partitions, + Consumer dropData) { dropPartitionMetadata(partitions); try (FileStoreCommit commit = table.store() @@ -150,7 +180,7 @@ public void dropPartitions(List> partitions) createCommitUser( table.coreOptions().toConfiguration()), null)) { - commit.dropPartitions(partitions, BatchWriteBuilder.COMMIT_IDENTIFIER); + dropData.accept(commit); } // Model HiveCatalog's cleanup of markers after deleting ordinary data. List> donePartitions = new ArrayList<>(); @@ -440,11 +470,14 @@ public void testExpireTimestampPartitionWithCustomFormatter(boolean legacyPartit assertThat(readTimestamps()).containsExactly(retainedTs); } - @Test - public void testExpireTimestampPartitionNamedAsDefaultPartition() throws Exception { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testExpireTimestampPartitionNamedAsDefaultPartition(boolean metastorePartitioned) + throws Exception { // The epoch partition is stored under the same name as the NULL partition, so parsing // the name back would drop the NULL partition instead of the selected one. Map tableOptions = new HashMap<>(); + tableOptions.put(METASTORE_PARTITIONED_TABLE.key(), String.valueOf(metastorePartitioned)); tableOptions.put(CoreOptions.PARTITION_DEFAULT_NAME.key(), "1970-01-01 00:00:00"); tableOptions.put(CoreOptions.PARTITION_GENERATE_LEGACY_NAME.key(), "false"); createPartitionedTable( @@ -462,24 +495,28 @@ public void testExpireTimestampPartitionNamedAsDefaultPartition() throws Excepti } @ParameterizedTest - @ValueSource(booleans = {false, true}) - public void testExpireDropsSelectedLocalZonedTimestampPartition(boolean legacyPartitionName) - throws Exception { + @CsvSource({"false, false", "false, true", "true, false", "true, true"}) + public void testExpireDropsSelectedLocalZonedTimestampPartition( + boolean legacyPartitionName, boolean metastorePartitioned) throws Exception { TimeZone defaultZone = TimeZone.getDefault(); TimeZone.setDefault(TimeZone.getTimeZone("America/New_York")); try { // The LTZ key is not used for time extraction. Its reported value is the UTC // Timestamp#toString spelling, which a partition spec would parse back in the default // time zone and match another partition. + Map tableOptions = new HashMap<>(); + tableOptions.put( + CoreOptions.PARTITION_GENERATE_LEGACY_NAME.key(), + String.valueOf(legacyPartitionName)); + tableOptions.put( + METASTORE_PARTITIONED_TABLE.key(), String.valueOf(metastorePartitioned)); createPartitionedTable( RowType.of( DataTypes.TIMESTAMP(0), DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(0), VarCharType.STRING_TYPE), Arrays.asList("f0", "f1"), - Collections.singletonMap( - CoreOptions.PARTITION_GENERATE_LEGACY_NAME.key(), - String.valueOf(legacyPartitionName))); + tableOptions); Timestamp day = Timestamp.fromLocalDateTime(date(1)); write(GenericRow.of(day, day, BinaryString.fromString("00:00Z"))); write( @@ -516,7 +553,7 @@ private void createPartitionedTable( throws Exception { SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), path); Map tableOptions = new HashMap<>(extraOptions); - tableOptions.put(METASTORE_PARTITIONED_TABLE.key(), "false"); + tableOptions.putIfAbsent(METASTORE_PARTITIONED_TABLE.key(), "false"); schemaManager.createTable( new Schema(rowType.getFields(), partitionKeys, emptyList(), tableOptions, "")); newTable(); diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index e2ca397882fd..dbca5a130479 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -32,6 +32,7 @@ import org.apache.paimon.catalog.PropertyChange; import org.apache.paimon.catalog.TableMetadata; import org.apache.paimon.client.ClientPool; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.hive.pool.CachedClientPool; @@ -488,7 +489,43 @@ private void createPartitionsUtil( @Override public void dropPartitions(Identifier identifier, List> partitions) throws TableNotExistException { + dropPartitions( + identifier, + this.loadTableSchema(identifier), + partitions, + () -> super.dropPartitions(identifier, partitions)); + } + + @Override + public void dropBinaryPartitions(Identifier identifier, List partitions) + throws TableNotExistException { TableSchema schema = this.loadTableSchema(identifier); + CoreOptions options = CoreOptions.fromMap(schema.options()); + InternalRowPartitionComputer partitionComputer = + new InternalRowPartitionComputer( + options.partitionDefaultName(), + schema.logicalPartitionType(), + schema.partitionKeys().toArray(new String[0]), + options.legacyPartitionName()); + List> partitionSpecs = new ArrayList<>(partitions.size()); + for (BinaryRow partition : partitions) { + partitionSpecs.add(partitionComputer.generatePartValues(partition)); + } + // The metastore keeps partitions by their stored names, but the data is dropped by the + // partition rows: a stored name is not always parsed back to the same partition. + dropPartitions( + identifier, + schema, + partitionSpecs, + () -> super.dropBinaryPartitions(identifier, partitions)); + } + + private void dropPartitions( + Identifier identifier, + TableSchema schema, + List> partitions, + PartitionDataDropper dataDropper) + throws TableNotExistException { boolean tagToPart = CoreOptions.fromMap(schema.options()).tagToPartitionField() != null; boolean metastorePartitioned = metastorePartitioned(schema); if (metastorePartitioned) { @@ -500,13 +537,19 @@ public void dropPartitions(Identifier identifier, List> part schema); } if (!tagToPart) { - super.dropPartitions(identifier, partitions); + dataDropper.drop(); if (metastorePartitioned) { dropDonePartitions(identifier, partitions, schema); } } } + /** Drops the data of the partitions after their metastore partitions. */ + @FunctionalInterface + private interface PartitionDataDropper { + void drop() throws TableNotExistException; + } + private void dropDonePartitions( Identifier identifier, List> partitions, TableSchema schema) throws TableNotExistException { diff --git a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java index ea8043171aba..7798596254ce 100644 --- a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java +++ b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java @@ -27,6 +27,7 @@ import org.apache.paimon.client.ClientPool; import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.Timestamp; import org.apache.paimon.fs.Path; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; @@ -43,6 +44,7 @@ import org.apache.paimon.table.sink.BatchTableCommit; import org.apache.paimon.table.sink.BatchTableWrite; import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.table.source.ReadBuilder; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.utils.CommonTestUtils; @@ -72,6 +74,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.TimeZone; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @@ -885,6 +888,122 @@ public void testExpireIntegerPartitionsWithDoneMarkers(boolean markDone) throws } } + @Test + public void testExpireTimestampPartitionNamedAsDefaultPartition() throws Exception { + // The epoch partition is stored under the same name as the NULL partition, so parsing the + // name back would drop the NULL partition instead of the selected one. + String databaseName = "test_expire_default_named_partition"; + catalog.createDatabase(databaseName, false); + Identifier identifier = Identifier.create(databaseName, "t"); + catalog.createTable( + identifier, + Schema.newBuilder() + .option(METASTORE_PARTITIONED_TABLE.key(), "true") + .option(CoreOptions.PARTITION_DEFAULT_NAME.key(), "1970-01-01 00:00:00") + .option(CoreOptions.PARTITION_GENERATE_LEGACY_NAME.key(), "false") + .option(PARTITION_TIMESTAMP_FORMATTER.key(), "yyyy-MM-dd'T'HH:mm") + .option(PARTITION_TIMESTAMP_PATTERN.key(), "$dt") + .column("col", DataTypes.STRING()) + .column("dt", DataTypes.TIMESTAMP(0)) + .partitionKeys("dt") + .build(), + false); + FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); + // Commit separately: both partitions share one metastore partition name. + for (GenericRow row : + Arrays.asList( + GenericRow.of( + BinaryString.fromString("epoch"), Timestamp.fromEpochMillis(0)), + GenericRow.of(BinaryString.fromString("null"), null))) { + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = writeBuilder.newWrite(); + BatchTableCommit commit = writeBuilder.newCommit()) { + write.write(row); + commit.commit(write.prepareCommit()); + } + } + + table = table.copy(valuesTimeExpirationOptions()); + table.store().newPartitionExpire("expire", table).expire(Long.MAX_VALUE); + + assertThat(readFirstColumn(table)).containsExactly("null"); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testExpireDropsSelectedLocalZonedTimestampPartition(boolean legacyPartitionName) + throws Exception { + TimeZone defaultZone = TimeZone.getDefault(); + TimeZone.setDefault(TimeZone.getTimeZone("America/New_York")); + try { + // The LTZ key is not used for time extraction. A legacy name of it is the UTC + // Timestamp#toString spelling, which a spec would parse back in the default time zone + // and match another partition. + String databaseName = "test_expire_ltz_sub_partition_" + legacyPartitionName; + catalog.createDatabase(databaseName, false); + Identifier identifier = Identifier.create(databaseName, "t"); + catalog.createTable( + identifier, + Schema.newBuilder() + .option(METASTORE_PARTITIONED_TABLE.key(), "true") + .option( + CoreOptions.PARTITION_GENERATE_LEGACY_NAME.key(), + String.valueOf(legacyPartitionName)) + .option(PARTITION_TIMESTAMP_PATTERN.key(), "$dt") + .option(CoreOptions.PARTITION_EXPIRATION_MAX_NUM.key(), "1") + .column("col", DataTypes.STRING()) + .column("dt", DataTypes.TIMESTAMP(0)) + .column("ts", DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(0)) + .partitionKeys("dt", "ts") + .build(), + false); + FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); + Timestamp day = Timestamp.fromEpochMillis(0); + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = writeBuilder.newWrite(); + BatchTableCommit commit = writeBuilder.newCommit()) { + write.write(GenericRow.of(BinaryString.fromString("00:00Z"), day, day)); + write.write( + GenericRow.of( + BinaryString.fromString("05:00Z"), + day, + Timestamp.fromEpochMillis(5 * 60 * 60 * 1000L))); + commit.commit(write.prepareCommit()); + } + + table = table.copy(valuesTimeExpirationOptions()); + table.store().newPartitionExpire("expire", table).expire(Long.MAX_VALUE); + + assertThat(readFirstColumn(table)).containsExactly("05:00Z"); + // the metastore partition is dropped by its stored name + assertThat( + ((HiveCatalog) catalog) + .getHmsClient() + .listPartitions(databaseName, "t", Short.MAX_VALUE)) + .hasSize(1); + } finally { + TimeZone.setDefault(defaultZone); + } + } + + private static Map valuesTimeExpirationOptions() { + Map options = new HashMap<>(); + options.put(CoreOptions.PARTITION_EXPIRATION_TIME.key(), "1 d"); + options.put(CoreOptions.PARTITION_EXPIRATION_CHECK_INTERVAL.key(), "0 s"); + options.put(CoreOptions.PARTITION_EXPIRATION_STRATEGY.key(), "values-time"); + return options; + } + + private static List readFirstColumn(FileStoreTable table) throws Exception { + List values = new ArrayList<>(); + ReadBuilder readBuilder = table.newReadBuilder(); + readBuilder + .newRead() + .createReader(readBuilder.newScan().plan()) + .forEachRemaining(row -> values.add(row.getString(0).toString())); + return values; + } + @Test public void testDropPartitionsCleansDoneMarkersInPartitionKeyOrder() throws Exception { String databaseName = "test_drop_multi_key_done_markers";