diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java index 037f7ea2bfba..81f2c8c714d4 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java @@ -20,6 +20,7 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.annotation.VisibleForTesting; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.fs.Path; import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; @@ -390,6 +391,15 @@ public void dropPartitions(Identifier identifier, List> part } } + @Override + public void dropBinaryPartitions(Identifier identifier, List partitions) + throws TableNotExistException { + wrapped.dropBinaryPartitions(identifier, partitions); + if (partitionCache != null) { + partitionCache.invalidate(identifier); + } + } + @Override public void alterPartitions(Identifier identifier, List partitions) throws TableNotExistException { diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java index 88390954d1ac..8fd4827a9b31 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java @@ -23,6 +23,7 @@ import org.apache.paimon.annotation.Public; import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.consumer.ConsumerInfo; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.function.Function; import org.apache.paimon.function.FunctionChange; import org.apache.paimon.operation.Lock; @@ -39,6 +40,7 @@ import org.apache.paimon.table.Table; import org.apache.paimon.table.TableSnapshot; import org.apache.paimon.table.sink.BatchTableCommit; +import org.apache.paimon.table.sink.TableCommitImpl; import org.apache.paimon.utils.SnapshotNotExistException; import org.apache.paimon.view.View; import org.apache.paimon.view.ViewChange; @@ -1155,6 +1157,38 @@ default void dropPartitions(Identifier identifier, List> par } } + /** + * Drop partitions of the specify table given by their partition rows. Ignore non-existent + * partitions. + * + *

Unlike {@link #dropPartitions(Identifier, List)}, the data is dropped by the exact + * partition rows rather than by partition specs parsed back from strings. A partition name is + * not always parsed back to the same partition, e.g. when it equals the default partition name + * or is a TIMESTAMP_LTZ value. Partition metadata kept by name (e.g. in a metastore) is still + * dropped by the stored partition names. + * + * @param identifier path of the table to drop partitions + * @param partitions partition rows to be deleted, in the partition type of the table + * @throws TableNotExistException if the table does not exist + */ + default void dropBinaryPartitions(Identifier identifier, List partitions) + throws TableNotExistException { + Table table = getTable(identifier); + try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) { + if (!(commit instanceof TableCommitImpl)) { + throw new UnsupportedOperationException( + String.format( + "Table %s does not support dropping partitions by partition rows.", + identifier)); + } + ((TableCommitImpl) commit).truncateBinaryPartitions(partitions); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + /** * Alter partitions of the specify table. For non-existent partitions, partitions will be * created directly. diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java index 8b0aa817d430..cd9fc2596f37 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java @@ -20,6 +20,7 @@ import org.apache.paimon.PagedList; import org.apache.paimon.Snapshot; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.function.Function; import org.apache.paimon.function.FunctionChange; import org.apache.paimon.operation.Lock; @@ -378,6 +379,12 @@ public void dropPartitions(Identifier identifier, List> part wrapped.dropPartitions(identifier, partitions); } + @Override + public void dropBinaryPartitions(Identifier identifier, List partitions) + throws TableNotExistException { + wrapped.dropBinaryPartitions(identifier, partitions); + } + @Override public void alterPartitions(Identifier identifier, List partitions) throws TableNotExistException { diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommit.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommit.java index 050a345a0e3b..b778d2285f2f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommit.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommit.java @@ -74,6 +74,17 @@ public interface FileStoreCommit extends AutoCloseable { */ void dropPartitions(List> partitions, long commitIdentifier); + /** + * Drop multiple partitions given by their exact partition values. Unlike {@link + * #dropPartitions}, the partitions are not parsed back from their names, so a name that is + * ambiguous (e.g. equal to the default partition name) or time-zone dependent still drops + * exactly the given partition. The {@link Snapshot.CommitKind} of generated snapshot is {@link + * Snapshot.CommitKind#OVERWRITE}. + * + * @param partitions A list of partitions. NOTE: cannot be empty! + */ + void dropBinaryPartitions(List partitions, long commitIdentifier); + void truncateTable(long commitIdentifier); /** Compact the manifest entries only. */ diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java index 860835de53da..5e7b251212e3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java @@ -716,6 +716,18 @@ public void dropPartitions(List> partitions, long commitIden partitionFilter, emptyList(), emptyList(), commitIdentifier, null, new HashMap<>()); } + @Override + public void dropBinaryPartitions(List partitions, long commitIdentifier) { + checkArgument(!partitions.isEmpty(), "Partitions list cannot be empty."); + tryOverwritePartition( + PartitionPredicate.fromMultiple(partitionType, partitions), + emptyList(), + emptyList(), + commitIdentifier, + null, + new HashMap<>()); + } + @Override public void truncateTable(long commitIdentifier) { tryOverwritePartition( diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/NormalPartitionExpire.java b/paimon-core/src/main/java/org/apache/paimon/operation/NormalPartitionExpire.java index 6abffab58d7e..e88f93a4201b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/NormalPartitionExpire.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/NormalPartitionExpire.java @@ -34,7 +34,6 @@ import javax.annotation.Nullable; import java.time.Duration; -import java.time.LocalDate; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Collection; @@ -157,63 +156,87 @@ private List> doExpire( LocalDateTime expireDateTime, long commitIdentifier) { List partitionEntries = strategy.selectExpiredPartitions(scan, expireDateTime); - List> expiredPartValues = new ArrayList<>(partitionEntries.size()); + List expiredPartitions = new ArrayList<>(partitionEntries.size()); for (PartitionEntry partition : partitionEntries) { Object[] array = strategy.convertPartition(partition.partition()); - // DATE values are normalized for time extraction; use the stored names for dropping. + // DATE and TIMESTAMP values are normalized for time extraction; report the stored + // names instead. Map partitionSpec = commit.pathFactory() .partitionComputer() .generatePartValues(partition.partition()); int index = 0; for (String value : partitionSpec.values()) { - if (array[index] instanceof LocalDate) { + if (array[index] != null && strategy.isNormalizedForTimeExtraction(index)) { array[index] = value; } index++; } - expiredPartValues.add(strategy.toPartitionValue(array)); + List values = strategy.toPartitionValue(array); + // Sort on the joined form, but build the partition from the original values: a + // value may itself contain DELIMITER, and splitting the joined string back apart + // would then shift every following field by one. + expiredPartitions.add( + new ExpiredPartition( + partition.partition(), + String.join(DELIMITER, values), + strategy.toPartitionString(values.toArray()))); } - List> expired = new ArrayList<>(); - if (!expiredPartValues.isEmpty()) { - // convert partition value to partition string, and limit the partition num - expired = convertToPartitionString(expiredPartValues); + List selected = + expiredPartitions.stream() + .sorted(Comparator.comparing(p -> p.sortKey)) + .limit(Math.min(expiredPartitions.size(), maxExpireNum)) + .collect(Collectors.toList()); + List> expired = + selected.stream().map(p -> p.name).collect(Collectors.toList()); + if (!selected.isEmpty()) { LOG.info("Expire Partitions: {}", expired); - if (expireBatchSize > 0 && expireBatchSize < expired.size()) { - Lists.partition(expired, expireBatchSize) + if (expireBatchSize > 0 && expireBatchSize < selected.size()) { + Lists.partition(selected, expireBatchSize) .forEach( expiredBatchPartitions -> doBatchExpire(expiredBatchPartitions, commitIdentifier)); } else { - doBatchExpire(expired, commitIdentifier); + doBatchExpire(selected, commitIdentifier); } } return expired; } private void doBatchExpire( - List> expiredBatchPartitions, long commitIdentifier) { + List expiredBatchPartitions, long commitIdentifier) { + // Drop the selected partitions themselves: a stored name is not always parsed back to the + // same partition, e.g. when it equals the default partition name or is a TIMESTAMP_LTZ + // value spelled in UTC but parsed in the local time zone. + List partitions = + expiredBatchPartitions.stream().map(p -> p.partition).collect(Collectors.toList()); if (partitionModification != null) { + // the metastore still drops its metadata by the stored names try { - partitionModification.dropPartitions(expiredBatchPartitions); + partitionModification.dropBinaryPartitions(partitions); } catch (Catalog.TableNotExistException e) { throw new RuntimeException(e); } } else { - commit.dropPartitions(expiredBatchPartitions, commitIdentifier); + commit.dropBinaryPartitions(partitions, commitIdentifier); } } - private List> convertToPartitionString( - List> expiredPartValues) { - // Sort on the joined form, but build the partition from the original values: a value - // may itself contain DELIMITER, and splitting the joined string back apart would then - // shift every following field by one. - return expiredPartValues.stream() - .sorted(Comparator.comparing(values -> String.join(DELIMITER, values))) - .map(values -> strategy.toPartitionString(values.toArray())) - .limit(Math.min(expiredPartValues.size(), maxExpireNum)) - .collect(Collectors.toList()); + /** + * A partition selected for expiration. The stored name is for reporting, while deletion uses + * the original partition. + */ + private static class ExpiredPartition { + + private final BinaryRow partition; + private final String sortKey; + private final Map name; + + private ExpiredPartition(BinaryRow partition, String sortKey, Map name) { + this.partition = partition; + this.sortKey = sortKey; + this.name = name; + } } } diff --git a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionExpireStrategy.java b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionExpireStrategy.java index fc00691c9a6a..bfc6e32f09c9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionExpireStrategy.java +++ b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionExpireStrategy.java @@ -22,10 +22,13 @@ import org.apache.paimon.catalog.CatalogLoader; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.Timestamp; import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.operation.FileStoreScan; -import org.apache.paimon.types.DataTypeRoot; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypeChecks; import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.DateTimeUtils; import org.apache.paimon.utils.RowDataToObjectArrayConverter; import org.apache.paimon.shade.guava30.com.google.common.primitives.UnsignedBytes; @@ -51,11 +54,20 @@ public abstract class PartitionExpireStrategy implements Serializable { protected final List partitionKeys; protected final String partitionDefaultName; private final RowDataToObjectArrayConverter toObjectArrayConverter; + @Nullable private final String timestampFormatter; public PartitionExpireStrategy(RowType partitionType, String partitionDefaultName) { + this(partitionType, partitionDefaultName, null); + } + + public PartitionExpireStrategy( + RowType partitionType, + String partitionDefaultName, + @Nullable String timestampFormatter) { this.toObjectArrayConverter = new RowDataToObjectArrayConverter(partitionType); this.partitionKeys = partitionType.getFieldNames(); this.partitionDefaultName = partitionDefaultName; + this.timestampFormatter = timestampFormatter; } public Map toPartitionString(Object[] array) { @@ -78,18 +90,53 @@ public List toPartitionValue(Object[] array) { return list; } + /** + * Whether {@link #convertPartition} renders the value at this index in a form meant for time + * extraction that can differ from the stored partition name. + */ + public boolean isNormalizedForTimeExtraction(int index) { + switch (toObjectArrayConverter.rowType().getTypeAt(index).getTypeRoot()) { + case DATE: + case TIMESTAMP_WITHOUT_TIME_ZONE: + return true; + default: + return false; + } + } + public Object[] convertPartition(BinaryRow partition) { Object[] values = toObjectArrayConverter.convert(partition); RowType partitionType = toObjectArrayConverter.rowType(); for (int index = 0; index < values.length; index++) { - if (values[index] != null - && partitionType.getTypeAt(index).getTypeRoot() == DataTypeRoot.DATE) { - values[index] = LocalDate.ofEpochDay((Integer) values[index]); + if (values[index] == null) { + continue; + } + DataType type = partitionType.getTypeAt(index); + switch (type.getTypeRoot()) { + case DATE: + values[index] = LocalDate.ofEpochDay((Integer) values[index]); + break; + case TIMESTAMP_WITHOUT_TIME_ZONE: + values[index] = + formatTimestampForTimeExtraction((Timestamp) values[index], type); + break; + default: + break; } } return values; } + private String formatTimestampForTimeExtraction(Timestamp timestamp, DataType type) { + if (timestampFormatter != null) { + // Keep the Timestamp#toString input an explicitly configured formatter expects. + return timestamp.toString(); + } + // Timestamp#toString uses the ISO 'T' separator, which the default partition time + // formatter cannot parse. Render it as the SQL timestamp string. + return DateTimeUtils.formatTimestamp(timestamp, DataTypeChecks.getPrecision(type)); + } + public abstract List selectExpiredPartitions( FileStoreScan scan, LocalDateTime expirationTime); diff --git a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionValuesTimeExpireStrategy.java b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionValuesTimeExpireStrategy.java index 0291bcc6c916..35a2b3a42f9e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionValuesTimeExpireStrategy.java +++ b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionValuesTimeExpireStrategy.java @@ -55,7 +55,7 @@ public class PartitionValuesTimeExpireStrategy extends PartitionExpireStrategy { private final boolean retainLatest; public PartitionValuesTimeExpireStrategy(CoreOptions options, RowType partitionType) { - super(partitionType, options.partitionDefaultName()); + super(partitionType, options.partitionDefaultName(), options.partitionTimestampFormatter()); String timePattern = options.partitionTimestampPattern(); String timeFormatter = options.partitionTimestampFormatter(); this.timeExtractor = new PartitionTimeExtractor(timePattern, timeFormatter); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/PartitionModification.java b/paimon-core/src/main/java/org/apache/paimon/table/PartitionModification.java index b3850bf9bef7..bad0ab74376e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/PartitionModification.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/PartitionModification.java @@ -20,6 +20,7 @@ import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.partition.PartitionStatistics; import java.util.List; @@ -33,6 +34,13 @@ void createPartitions(List> partitions) void dropPartitions(List> partitions) throws Catalog.TableNotExistException; + /** Drop partitions given by their partition rows, see {@link Catalog#dropBinaryPartitions}. */ + default void dropBinaryPartitions(List partitions) + throws Catalog.TableNotExistException { + throw new UnsupportedOperationException( + getClass().getName() + " does not support dropping partitions by partition rows."); + } + void alterPartitions(List partitions) throws Catalog.TableNotExistException; @@ -51,6 +59,12 @@ public void dropPartitions(List> partitions) catalog.dropPartitions(identifier, partitions); } + @Override + public void dropBinaryPartitions(List partitions) + throws Catalog.TableNotExistException { + catalog.dropBinaryPartitions(identifier, partitions); + } + @Override public void alterPartitions(List partitions) throws Catalog.TableNotExistException { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java index 23f1132fd795..832e0c01f594 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java @@ -218,6 +218,15 @@ public void truncatePartitions(List> partitionSpecs) { commit.dropPartitions(partitionSpecs, COMMIT_IDENTIFIER); } + /** + * Truncate the given partitions. Unlike {@link #truncatePartitions}, the partitions are not + * parsed back from their names, see {@link FileStoreCommit#dropBinaryPartitions}. + */ + public void truncateBinaryPartitions(List partitions) { + commit.withOperation(Snapshot.Operation.TRUNCATE); + commit.dropBinaryPartitions(partitions, COMMIT_IDENTIFIER); + } + @Override public void updateStatistics(Statistics statistics) { commit.commitStatistics(statistics, COMMIT_IDENTIFIER); diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java index 4b07a3fbbf46..0028aedfafc0 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java @@ -24,6 +24,7 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.Timestamp; import org.apache.paimon.data.serializer.InternalRowSerializer; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; @@ -67,6 +68,8 @@ import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; +import javax.annotation.Nullable; + import java.io.IOException; import java.time.LocalDate; import java.time.LocalDateTime; @@ -83,8 +86,10 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TimeZone; import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; +import java.util.function.Consumer; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; @@ -139,6 +144,35 @@ public void createPartitions(List> partitions) @Override public void dropPartitions(List> partitions) throws Catalog.TableNotExistException { + dropPartitions( + partitions, + commit -> + commit.dropPartitions( + partitions, BatchWriteBuilder.COMMIT_IDENTIFIER)); + } + + @Override + public void dropBinaryPartitions(List partitions) + throws Catalog.TableNotExistException { + // Model HiveCatalog: metadata by the stored names, data by the rows. + List> partitionSpecs = new ArrayList<>(); + for (BinaryRow partition : partitions) { + partitionSpecs.add( + table.store() + .pathFactory() + .partitionComputer() + .generatePartValues(partition)); + } + dropPartitions( + partitionSpecs, + commit -> + commit.dropBinaryPartitions( + partitions, BatchWriteBuilder.COMMIT_IDENTIFIER)); + } + + private void dropPartitions( + List> partitions, + Consumer dropData) { dropPartitionMetadata(partitions); try (FileStoreCommit commit = table.store() @@ -146,7 +180,7 @@ public void dropPartitions(List> partitions) createCommitUser( table.coreOptions().toConfiguration()), null)) { - commit.dropPartitions(partitions, BatchWriteBuilder.COMMIT_IDENTIFIER); + dropData.accept(commit); } // Model HiveCatalog's cleanup of markers after deleting ordinary data. List> donePartitions = new ArrayList<>(); @@ -337,6 +371,231 @@ public void testDateAndHourPartitionValues() { assertThat(strategy.isExpired(partitionDate.atTime(13, 0), partition)).isTrue(); } + @ParameterizedTest + @CsvSource({ + "values-time, false", + "values-time, true", + "update-time, false", + "update-time, true" + }) + public void testExpireTimestampPartitionWithoutMetastore( + String expirationStrategy, boolean legacyPartitionName) throws Exception { + SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), path); + Map tableOptions = new HashMap<>(); + tableOptions.put(METASTORE_PARTITIONED_TABLE.key(), "false"); + tableOptions.put( + CoreOptions.PARTITION_GENERATE_LEGACY_NAME.key(), + String.valueOf(legacyPartitionName)); + schemaManager.createTable( + new Schema( + RowType.of(DataTypes.TIMESTAMP(0), VarCharType.STRING_TYPE).getFields(), + singletonList("f0"), + emptyList(), + tableOptions, + "")); + newTable(); + Timestamp expiredTs = Timestamp.fromLocalDateTime(date(1)); + Timestamp retainedTs = Timestamp.fromLocalDateTime(date(5)); + write(GenericRow.of(expiredTs, BinaryString.fromString("expired"))); + write(GenericRow.of(retainedTs, BinaryString.fromString("retained"))); + + Map options = new HashMap<>(); + options.put(PARTITION_EXPIRATION_TIME.key(), "2 d"); + options.put(CoreOptions.PARTITION_EXPIRATION_STRATEGY.key(), expirationStrategy); + options.put(CoreOptions.PARTITION_TIMESTAMP_PATTERN.key(), "$f0"); + table = table.copy(options); + NormalPartitionExpire expire = + (NormalPartitionExpire) table.store().newPartitionExpire("", table); + expire.setLastCheck(date(1)); + // the legacy name is Timestamp#toString, the new name is the SQL timestamp string + String expiredName = legacyPartitionName ? "2023-01-01T00:00" : "2023-01-01 00:00:00"; + String retainedName = legacyPartitionName ? "2023-01-05T00:00" : "2023-01-05 00:00:00"; + List> expectedExpired = new ArrayList<>(); + expectedExpired.add(Collections.singletonMap("f0", expiredName)); + LocalDateTime checkTime = date(6); + List expectedRemaining = singletonList(retainedTs); + if ("update-time".equals(expirationStrategy)) { + checkTime = LocalDateTime.now().plusDays(3); + expectedExpired.add(Collections.singletonMap("f0", retainedName)); + expectedRemaining = emptyList(); + } + assertThat(expire.expire(checkTime, Long.MAX_VALUE)) + .containsExactlyElementsOf(expectedExpired); + List remaining = new ArrayList<>(); + table.newRead() + .createReader(table.newScan().plan().splits()) + .forEachRemaining(row -> remaining.add(row.getTimestamp(0, 0))); + assertThat(remaining).containsExactlyElementsOf(expectedRemaining); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testLocalZonedTimestampPartitionIsNotExpiredByDefault(boolean legacyPartitionName) + throws Exception { + TimeZone defaultZone = TimeZone.getDefault(); + TimeZone.setDefault(TimeZone.getTimeZone("America/New_York")); + try { + // TIMESTAMP_LTZ values are not normalized for time extraction, so their ISO spelling + // is still not parsed by the default extractor and they are not expired + createTimestampPartitionedTable( + DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(0), legacyPartitionName); + Timestamp oldTs = Timestamp.fromLocalDateTime(date(1)); + Timestamp freshTs = Timestamp.fromLocalDateTime(date(1).plusHours(5)); + write(GenericRow.of(oldTs, BinaryString.fromString("old"))); + write(GenericRow.of(freshTs, BinaryString.fromString("fresh"))); + + NormalPartitionExpire expire = newValuesTimeExpire("1 d", null); + assertThat(expire.expire(date(1).plusHours(22), Long.MAX_VALUE)).isEmpty(); + assertThat(readTimestamps()).containsExactlyInAnyOrder(oldTs, freshTs); + } finally { + TimeZone.setDefault(defaultZone); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testExpireTimestampPartitionWithCustomFormatter(boolean legacyPartitionName) + throws Exception { + createTimestampPartitionedTable(DataTypes.TIMESTAMP(0), legacyPartitionName); + Timestamp expiredTs = Timestamp.fromLocalDateTime(date(1)); + Timestamp retainedTs = Timestamp.fromLocalDateTime(date(5)); + write(GenericRow.of(expiredTs, BinaryString.fromString("expired"))); + write(GenericRow.of(retainedTs, BinaryString.fromString("retained"))); + + // the formatter matches the Timestamp#toString input it has always received + NormalPartitionExpire expire = newValuesTimeExpire("2 d", "yyyy-MM-dd'T'HH:mm"); + String expiredName = legacyPartitionName ? "2023-01-01T00:00" : "2023-01-01 00:00:00"; + assertThat(expire.expire(date(6), Long.MAX_VALUE)) + .containsExactly(Collections.singletonMap("f0", expiredName)); + assertThat(readTimestamps()).containsExactly(retainedTs); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testExpireTimestampPartitionNamedAsDefaultPartition(boolean metastorePartitioned) + throws Exception { + // The epoch partition is stored under the same name as the NULL partition, so parsing + // the name back would drop the NULL partition instead of the selected one. + Map tableOptions = new HashMap<>(); + tableOptions.put(METASTORE_PARTITIONED_TABLE.key(), String.valueOf(metastorePartitioned)); + tableOptions.put(CoreOptions.PARTITION_DEFAULT_NAME.key(), "1970-01-01 00:00:00"); + tableOptions.put(CoreOptions.PARTITION_GENERATE_LEGACY_NAME.key(), "false"); + createPartitionedTable( + RowType.of(DataTypes.TIMESTAMP(0), VarCharType.STRING_TYPE), + singletonList("f0"), + tableOptions); + write(GenericRow.of(Timestamp.fromEpochMillis(0), BinaryString.fromString("epoch"))); + write(GenericRow.of(null, BinaryString.fromString("null"))); + + NormalPartitionExpire expire = + newValuesTimeExpire("2 d", "yyyy-MM-dd'T'HH:mm", Collections.emptyMap()); + assertThat(expire.expire(date(6), Long.MAX_VALUE)) + .containsExactly(Collections.singletonMap("f0", "1970-01-01 00:00:00")); + assertThat(readStrings(1)).containsExactly("null"); + } + + @ParameterizedTest + @CsvSource({"false, false", "false, true", "true, false", "true, true"}) + public void testExpireDropsSelectedLocalZonedTimestampPartition( + boolean legacyPartitionName, boolean metastorePartitioned) throws Exception { + TimeZone defaultZone = TimeZone.getDefault(); + TimeZone.setDefault(TimeZone.getTimeZone("America/New_York")); + try { + // The LTZ key is not used for time extraction. Its reported value is the UTC + // Timestamp#toString spelling, which a partition spec would parse back in the default + // time zone and match another partition. + Map tableOptions = new HashMap<>(); + tableOptions.put( + CoreOptions.PARTITION_GENERATE_LEGACY_NAME.key(), + String.valueOf(legacyPartitionName)); + tableOptions.put( + METASTORE_PARTITIONED_TABLE.key(), String.valueOf(metastorePartitioned)); + createPartitionedTable( + RowType.of( + DataTypes.TIMESTAMP(0), + DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(0), + VarCharType.STRING_TYPE), + Arrays.asList("f0", "f1"), + tableOptions); + Timestamp day = Timestamp.fromLocalDateTime(date(1)); + write(GenericRow.of(day, day, BinaryString.fromString("00:00Z"))); + write( + GenericRow.of( + day, + Timestamp.fromLocalDateTime(date(1).plusHours(5)), + BinaryString.fromString("05:00Z"))); + + NormalPartitionExpire expire = + newValuesTimeExpire( + "2 d", + null, + Collections.singletonMap( + CoreOptions.PARTITION_EXPIRATION_MAX_NUM.key(), "1")); + assertThat(expire.expire(date(6), Long.MAX_VALUE)).hasSize(1); + assertThat(readStrings(2)).containsExactly("05:00Z"); + } finally { + TimeZone.setDefault(defaultZone); + } + } + + private void createTimestampPartitionedTable(DataType partitionType, boolean legacyName) + throws Exception { + createPartitionedTable( + RowType.of(partitionType, VarCharType.STRING_TYPE), + singletonList("f0"), + Collections.singletonMap( + CoreOptions.PARTITION_GENERATE_LEGACY_NAME.key(), + String.valueOf(legacyName))); + } + + private void createPartitionedTable( + RowType rowType, List partitionKeys, Map extraOptions) + throws Exception { + SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), path); + Map tableOptions = new HashMap<>(extraOptions); + tableOptions.putIfAbsent(METASTORE_PARTITIONED_TABLE.key(), "false"); + schemaManager.createTable( + new Schema(rowType.getFields(), partitionKeys, emptyList(), tableOptions, "")); + newTable(); + } + + private List readStrings(int index) throws Exception { + List values = new ArrayList<>(); + table.newRead() + .createReader(table.newScan().plan().splits()) + .forEachRemaining(row -> values.add(row.getString(index).toString())); + return values; + } + + private NormalPartitionExpire newValuesTimeExpire( + String expirationTime, @Nullable String formatter) { + return newValuesTimeExpire(expirationTime, formatter, Collections.emptyMap()); + } + + private NormalPartitionExpire newValuesTimeExpire( + String expirationTime, @Nullable String formatter, Map extraOptions) { + Map options = new HashMap<>(extraOptions); + options.put(PARTITION_EXPIRATION_TIME.key(), expirationTime); + options.put(CoreOptions.PARTITION_EXPIRATION_STRATEGY.key(), "values-time"); + options.put(CoreOptions.PARTITION_TIMESTAMP_PATTERN.key(), "$f0"); + if (formatter != null) { + options.put(CoreOptions.PARTITION_TIMESTAMP_FORMATTER.key(), formatter); + } + table = table.copy(options); + NormalPartitionExpire expire = + (NormalPartitionExpire) table.store().newPartitionExpire("", table); + expire.setLastCheck(date(1)); + return expire; + } + + private List readTimestamps() throws Exception { + List timestamps = new ArrayList<>(); + table.newRead() + .createReader(table.newScan().plan().splits()) + .forEachRemaining(row -> timestamps.add(row.getTimestamp(0, 0))); + return timestamps; + } + @ParameterizedTest @ValueSource(strings = {"values-time", "update-time"}) public void testExpireDatePartitionWithoutMetastore(String expirationStrategy) diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index e2ca397882fd..dbca5a130479 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -32,6 +32,7 @@ import org.apache.paimon.catalog.PropertyChange; import org.apache.paimon.catalog.TableMetadata; import org.apache.paimon.client.ClientPool; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.hive.pool.CachedClientPool; @@ -488,7 +489,43 @@ private void createPartitionsUtil( @Override public void dropPartitions(Identifier identifier, List> partitions) throws TableNotExistException { + dropPartitions( + identifier, + this.loadTableSchema(identifier), + partitions, + () -> super.dropPartitions(identifier, partitions)); + } + + @Override + public void dropBinaryPartitions(Identifier identifier, List partitions) + throws TableNotExistException { TableSchema schema = this.loadTableSchema(identifier); + CoreOptions options = CoreOptions.fromMap(schema.options()); + InternalRowPartitionComputer partitionComputer = + new InternalRowPartitionComputer( + options.partitionDefaultName(), + schema.logicalPartitionType(), + schema.partitionKeys().toArray(new String[0]), + options.legacyPartitionName()); + List> partitionSpecs = new ArrayList<>(partitions.size()); + for (BinaryRow partition : partitions) { + partitionSpecs.add(partitionComputer.generatePartValues(partition)); + } + // The metastore keeps partitions by their stored names, but the data is dropped by the + // partition rows: a stored name is not always parsed back to the same partition. + dropPartitions( + identifier, + schema, + partitionSpecs, + () -> super.dropBinaryPartitions(identifier, partitions)); + } + + private void dropPartitions( + Identifier identifier, + TableSchema schema, + List> partitions, + PartitionDataDropper dataDropper) + throws TableNotExistException { boolean tagToPart = CoreOptions.fromMap(schema.options()).tagToPartitionField() != null; boolean metastorePartitioned = metastorePartitioned(schema); if (metastorePartitioned) { @@ -500,13 +537,19 @@ public void dropPartitions(Identifier identifier, List> part schema); } if (!tagToPart) { - super.dropPartitions(identifier, partitions); + dataDropper.drop(); if (metastorePartitioned) { dropDonePartitions(identifier, partitions, schema); } } } + /** Drops the data of the partitions after their metastore partitions. */ + @FunctionalInterface + private interface PartitionDataDropper { + void drop() throws TableNotExistException; + } + private void dropDonePartitions( Identifier identifier, List> partitions, TableSchema schema) throws TableNotExistException { diff --git a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java index ea8043171aba..7798596254ce 100644 --- a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java +++ b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java @@ -27,6 +27,7 @@ import org.apache.paimon.client.ClientPool; import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.Timestamp; import org.apache.paimon.fs.Path; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; @@ -43,6 +44,7 @@ import org.apache.paimon.table.sink.BatchTableCommit; import org.apache.paimon.table.sink.BatchTableWrite; import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.table.source.ReadBuilder; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.utils.CommonTestUtils; @@ -72,6 +74,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.TimeZone; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @@ -885,6 +888,122 @@ public void testExpireIntegerPartitionsWithDoneMarkers(boolean markDone) throws } } + @Test + public void testExpireTimestampPartitionNamedAsDefaultPartition() throws Exception { + // The epoch partition is stored under the same name as the NULL partition, so parsing the + // name back would drop the NULL partition instead of the selected one. + String databaseName = "test_expire_default_named_partition"; + catalog.createDatabase(databaseName, false); + Identifier identifier = Identifier.create(databaseName, "t"); + catalog.createTable( + identifier, + Schema.newBuilder() + .option(METASTORE_PARTITIONED_TABLE.key(), "true") + .option(CoreOptions.PARTITION_DEFAULT_NAME.key(), "1970-01-01 00:00:00") + .option(CoreOptions.PARTITION_GENERATE_LEGACY_NAME.key(), "false") + .option(PARTITION_TIMESTAMP_FORMATTER.key(), "yyyy-MM-dd'T'HH:mm") + .option(PARTITION_TIMESTAMP_PATTERN.key(), "$dt") + .column("col", DataTypes.STRING()) + .column("dt", DataTypes.TIMESTAMP(0)) + .partitionKeys("dt") + .build(), + false); + FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); + // Commit separately: both partitions share one metastore partition name. + for (GenericRow row : + Arrays.asList( + GenericRow.of( + BinaryString.fromString("epoch"), Timestamp.fromEpochMillis(0)), + GenericRow.of(BinaryString.fromString("null"), null))) { + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = writeBuilder.newWrite(); + BatchTableCommit commit = writeBuilder.newCommit()) { + write.write(row); + commit.commit(write.prepareCommit()); + } + } + + table = table.copy(valuesTimeExpirationOptions()); + table.store().newPartitionExpire("expire", table).expire(Long.MAX_VALUE); + + assertThat(readFirstColumn(table)).containsExactly("null"); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testExpireDropsSelectedLocalZonedTimestampPartition(boolean legacyPartitionName) + throws Exception { + TimeZone defaultZone = TimeZone.getDefault(); + TimeZone.setDefault(TimeZone.getTimeZone("America/New_York")); + try { + // The LTZ key is not used for time extraction. A legacy name of it is the UTC + // Timestamp#toString spelling, which a spec would parse back in the default time zone + // and match another partition. + String databaseName = "test_expire_ltz_sub_partition_" + legacyPartitionName; + catalog.createDatabase(databaseName, false); + Identifier identifier = Identifier.create(databaseName, "t"); + catalog.createTable( + identifier, + Schema.newBuilder() + .option(METASTORE_PARTITIONED_TABLE.key(), "true") + .option( + CoreOptions.PARTITION_GENERATE_LEGACY_NAME.key(), + String.valueOf(legacyPartitionName)) + .option(PARTITION_TIMESTAMP_PATTERN.key(), "$dt") + .option(CoreOptions.PARTITION_EXPIRATION_MAX_NUM.key(), "1") + .column("col", DataTypes.STRING()) + .column("dt", DataTypes.TIMESTAMP(0)) + .column("ts", DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(0)) + .partitionKeys("dt", "ts") + .build(), + false); + FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); + Timestamp day = Timestamp.fromEpochMillis(0); + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = writeBuilder.newWrite(); + BatchTableCommit commit = writeBuilder.newCommit()) { + write.write(GenericRow.of(BinaryString.fromString("00:00Z"), day, day)); + write.write( + GenericRow.of( + BinaryString.fromString("05:00Z"), + day, + Timestamp.fromEpochMillis(5 * 60 * 60 * 1000L))); + commit.commit(write.prepareCommit()); + } + + table = table.copy(valuesTimeExpirationOptions()); + table.store().newPartitionExpire("expire", table).expire(Long.MAX_VALUE); + + assertThat(readFirstColumn(table)).containsExactly("05:00Z"); + // the metastore partition is dropped by its stored name + assertThat( + ((HiveCatalog) catalog) + .getHmsClient() + .listPartitions(databaseName, "t", Short.MAX_VALUE)) + .hasSize(1); + } finally { + TimeZone.setDefault(defaultZone); + } + } + + private static Map valuesTimeExpirationOptions() { + Map options = new HashMap<>(); + options.put(CoreOptions.PARTITION_EXPIRATION_TIME.key(), "1 d"); + options.put(CoreOptions.PARTITION_EXPIRATION_CHECK_INTERVAL.key(), "0 s"); + options.put(CoreOptions.PARTITION_EXPIRATION_STRATEGY.key(), "values-time"); + return options; + } + + private static List readFirstColumn(FileStoreTable table) throws Exception { + List values = new ArrayList<>(); + ReadBuilder readBuilder = table.newReadBuilder(); + readBuilder + .newRead() + .createReader(readBuilder.newScan().plan()) + .forEachRemaining(row -> values.add(row.getString(0).toString())); + return values; + } + @Test public void testDropPartitionsCleansDoneMarkersInPartitionKeyOrder() throws Exception { String databaseName = "test_drop_multi_key_done_markers";