Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import org.apache.paimon.CoreOptions;
import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.fs.Path;
import org.apache.paimon.options.MemorySize;
import org.apache.paimon.options.Options;
Expand Down Expand Up @@ -390,6 +391,15 @@ public void dropPartitions(Identifier identifier, List<Map<String, String>> part
}
}

@Override
public void dropBinaryPartitions(Identifier identifier, List<BinaryRow> partitions)
throws TableNotExistException {
wrapped.dropBinaryPartitions(identifier, partitions);
if (partitionCache != null) {
partitionCache.invalidate(identifier);
}
}

@Override
public void alterPartitions(Identifier identifier, List<PartitionStatistics> partitions)
throws TableNotExistException {
Expand Down
34 changes: 34 additions & 0 deletions paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.apache.paimon.annotation.Public;
import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.consumer.ConsumerInfo;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.function.Function;
import org.apache.paimon.function.FunctionChange;
import org.apache.paimon.operation.Lock;
Expand All @@ -39,6 +40,7 @@
import org.apache.paimon.table.Table;
import org.apache.paimon.table.TableSnapshot;
import org.apache.paimon.table.sink.BatchTableCommit;
import org.apache.paimon.table.sink.TableCommitImpl;
import org.apache.paimon.utils.SnapshotNotExistException;
import org.apache.paimon.view.View;
import org.apache.paimon.view.ViewChange;
Expand Down Expand Up @@ -1155,6 +1157,38 @@ default void dropPartitions(Identifier identifier, List<Map<String, String>> par
}
}

/**
* Drop partitions of the specify table given by their partition rows. Ignore non-existent
* partitions.
*
* <p>Unlike {@link #dropPartitions(Identifier, List)}, the data is dropped by the exact
* partition rows rather than by partition specs parsed back from strings. A partition name is
* not always parsed back to the same partition, e.g. when it equals the default partition name
* or is a TIMESTAMP_LTZ value. Partition metadata kept by name (e.g. in a metastore) is still
* dropped by the stored partition names.
*
* @param identifier path of the table to drop partitions
* @param partitions partition rows to be deleted, in the partition type of the table
* @throws TableNotExistException if the table does not exist
*/
default void dropBinaryPartitions(Identifier identifier, List<BinaryRow> partitions)
throws TableNotExistException {
Table table = getTable(identifier);
try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) {
if (!(commit instanceof TableCommitImpl)) {
throw new UnsupportedOperationException(
String.format(
"Table %s does not support dropping partitions by partition rows.",
identifier));
}
((TableCommitImpl) commit).truncateBinaryPartitions(partitions);
} catch (RuntimeException e) {
throw e;
} catch (Exception e) {
throw new RuntimeException(e);
}
}

/**
* Alter partitions of the specify table. For non-existent partitions, partitions will be
* created directly.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import org.apache.paimon.PagedList;
import org.apache.paimon.Snapshot;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.function.Function;
import org.apache.paimon.function.FunctionChange;
import org.apache.paimon.operation.Lock;
Expand Down Expand Up @@ -378,6 +379,12 @@ public void dropPartitions(Identifier identifier, List<Map<String, String>> part
wrapped.dropPartitions(identifier, partitions);
}

@Override
public void dropBinaryPartitions(Identifier identifier, List<BinaryRow> partitions)
throws TableNotExistException {
wrapped.dropBinaryPartitions(identifier, partitions);
}

@Override
public void alterPartitions(Identifier identifier, List<PartitionStatistics> partitions)
throws TableNotExistException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,17 @@ public interface FileStoreCommit extends AutoCloseable {
*/
void dropPartitions(List<Map<String, String>> partitions, long commitIdentifier);

/**
* Drop multiple partitions given by their exact partition values. Unlike {@link
* #dropPartitions}, the partitions are not parsed back from their names, so a name that is
* ambiguous (e.g. equal to the default partition name) or time-zone dependent still drops
* exactly the given partition. The {@link Snapshot.CommitKind} of generated snapshot is {@link
* Snapshot.CommitKind#OVERWRITE}.
*
* @param partitions A list of partitions. NOTE: cannot be empty!
*/
void dropBinaryPartitions(List<BinaryRow> partitions, long commitIdentifier);

void truncateTable(long commitIdentifier);

/** Compact the manifest entries only. */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -716,6 +716,18 @@ public void dropPartitions(List<Map<String, String>> partitions, long commitIden
partitionFilter, emptyList(), emptyList(), commitIdentifier, null, new HashMap<>());
}

@Override
public void dropBinaryPartitions(List<BinaryRow> partitions, long commitIdentifier) {
checkArgument(!partitions.isEmpty(), "Partitions list cannot be empty.");
tryOverwritePartition(
PartitionPredicate.fromMultiple(partitionType, partitions),
emptyList(),
emptyList(),
commitIdentifier,
null,
new HashMap<>());
}

@Override
public void truncateTable(long commitIdentifier) {
tryOverwritePartition(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
import javax.annotation.Nullable;

import java.time.Duration;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Collection;
Expand Down Expand Up @@ -157,63 +156,87 @@ private List<Map<String, String>> doExpire(
LocalDateTime expireDateTime, long commitIdentifier) {
List<PartitionEntry> partitionEntries =
strategy.selectExpiredPartitions(scan, expireDateTime);
List<List<String>> expiredPartValues = new ArrayList<>(partitionEntries.size());
List<ExpiredPartition> expiredPartitions = new ArrayList<>(partitionEntries.size());
for (PartitionEntry partition : partitionEntries) {
Object[] array = strategy.convertPartition(partition.partition());
// DATE values are normalized for time extraction; use the stored names for dropping.
// DATE and TIMESTAMP values are normalized for time extraction; report the stored
// names instead.
Map<String, String> partitionSpec =
commit.pathFactory()
.partitionComputer()
.generatePartValues(partition.partition());
int index = 0;
for (String value : partitionSpec.values()) {
if (array[index] instanceof LocalDate) {
if (array[index] != null && strategy.isNormalizedForTimeExtraction(index)) {
array[index] = value;

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P1] Preserve the LTZ partition key when dropping legacy names

The legacy LTZ name is a UTC Timestamp.toString() spelling, but commit.dropPartitions reparses the supplied string as local time. Replacing the normalized value with this name therefore targets a different key. I reproduced actual writes/expiry/readback with TIMESTAMP_LTZ(0), legacy partition names and America/New_York: write instants 2023-01-01T00:00Z and 2023-01-01T05:00Z, then expire at local January 1 22:00 with one-day retention. The returned selection correctly contains only legacy name 2023-01-01T00:00 (the 27-hour-old partition), but readback leaves that old row and removes the 05:00Z row, which is only 22 hours old. The legacy name was reparsed as midnight New York, i.e. 05:00Z. The new normalization activates this deletion path where the old default extractor skipped the ISO values. Please make truncation round-trip the original typed LTZ key (or use a legacy-aware conversion), keeping stored/HMS spelling separate where necessary, and cover a negative-offset timezone with two such partitions. This is a logical deletion of the wrong live partition, not a claim of immediate physical file erasure.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P1] Preserve the selected typed partition key when deleting locally. The generated path spelling is not always safe to parse back as a partition spec. With current names, partition.default-name='1970-01-01 00:00:00', an NTZ epoch partition, a NULL partition, and the existing ISO custom formatter, this head selects the epoch partition but deletes the NULL row. The same test against the base deletes epoch and retains NULL. A second real-table case uses (NTZ, LTZ) keys, the default extractor, New York timezone, and an expiration cap of 1. It reports the LTZ 00:00Z partition but deletes the 05:00Z partition because the LTZ ISO string is reparsed in local time. Keep stored/HMS names separate from the original BinaryRow used for local deletion, and cover both readback cases.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Rechecked b79af8f: the new binary-key deletion fixes the local path, and 156 expiration/extractor/commit tests pass with normal JDK 8 Maven checks. The metastore path still has the same P1: doBatchExpire delegates names to partitionModification.dropPartitions, which reaches HiveCatalog.dropPartitions -> super.dropPartitions -> FileStoreCommit.dropPartitions and reparses the key.

This reproduces using the PR's own new tests: set metastore.partitioned-table=true in createPartitionedTable. testExpireTimestampPartitionNamedAsDefaultPartition leaves the epoch row and deletes NULL; both testExpireDropsSelectedLocalZonedTimestampPartition naming modes leave 00:00Z and delete 05:00Z. These are actual commit/expiration/readback tests using the existing catalog callback, not just checks of returned partition names.

I also ran the epoch/NULL case with the three expiration production files restored to merge-base d7b521d: it deletes epoch and retains NULL. The identical data assertion fails on this head (only the returned-name assertion is omitted in both runs because base reports ISO and head reports SQL). Please propagate the exact selected typed key through the metastore-backed data deletion path too, while retaining stored names for HMS metadata operations; selecting the right key only in the local branch leaves this regression active.

}
index++;
}
expiredPartValues.add(strategy.toPartitionValue(array));
List<String> values = strategy.toPartitionValue(array);
// Sort on the joined form, but build the partition from the original values: a
// value may itself contain DELIMITER, and splitting the joined string back apart
// would then shift every following field by one.
expiredPartitions.add(
new ExpiredPartition(
partition.partition(),
String.join(DELIMITER, values),
strategy.toPartitionString(values.toArray())));
}

List<Map<String, String>> expired = new ArrayList<>();
if (!expiredPartValues.isEmpty()) {
// convert partition value to partition string, and limit the partition num
expired = convertToPartitionString(expiredPartValues);
List<ExpiredPartition> selected =
expiredPartitions.stream()
.sorted(Comparator.comparing(p -> p.sortKey))
.limit(Math.min(expiredPartitions.size(), maxExpireNum))
.collect(Collectors.toList());
List<Map<String, String>> expired =
selected.stream().map(p -> p.name).collect(Collectors.toList());
if (!selected.isEmpty()) {
LOG.info("Expire Partitions: {}", expired);
if (expireBatchSize > 0 && expireBatchSize < expired.size()) {
Lists.partition(expired, expireBatchSize)
if (expireBatchSize > 0 && expireBatchSize < selected.size()) {
Lists.partition(selected, expireBatchSize)
.forEach(
expiredBatchPartitions ->
doBatchExpire(expiredBatchPartitions, commitIdentifier));
} else {
doBatchExpire(expired, commitIdentifier);
doBatchExpire(selected, commitIdentifier);
}
}
return expired;
}

private void doBatchExpire(
List<Map<String, String>> expiredBatchPartitions, long commitIdentifier) {
List<ExpiredPartition> expiredBatchPartitions, long commitIdentifier) {
// Drop the selected partitions themselves: a stored name is not always parsed back to the
// same partition, e.g. when it equals the default partition name or is a TIMESTAMP_LTZ
// value spelled in UTC but parsed in the local time zone.
List<BinaryRow> partitions =
expiredBatchPartitions.stream().map(p -> p.partition).collect(Collectors.toList());
if (partitionModification != null) {
// the metastore still drops its metadata by the stored names
try {
partitionModification.dropPartitions(expiredBatchPartitions);
partitionModification.dropBinaryPartitions(partitions);
} catch (Catalog.TableNotExistException e) {
throw new RuntimeException(e);
}
} else {
commit.dropPartitions(expiredBatchPartitions, commitIdentifier);
commit.dropBinaryPartitions(partitions, commitIdentifier);
}
}

private List<Map<String, String>> convertToPartitionString(
List<List<String>> expiredPartValues) {
// Sort on the joined form, but build the partition from the original values: a value
// may itself contain DELIMITER, and splitting the joined string back apart would then
// shift every following field by one.
return expiredPartValues.stream()
.sorted(Comparator.comparing(values -> String.join(DELIMITER, values)))
.map(values -> strategy.toPartitionString(values.toArray()))
.limit(Math.min(expiredPartValues.size(), maxExpireNum))
.collect(Collectors.toList());
/**
* A partition selected for expiration. The stored name is for reporting, while deletion uses
* the original partition.
*/
private static class ExpiredPartition {

private final BinaryRow partition;
private final String sortKey;
private final Map<String, String> name;

private ExpiredPartition(BinaryRow partition, String sortKey, Map<String, String> name) {
this.partition = partition;
this.sortKey = sortKey;
this.name = name;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,13 @@
import org.apache.paimon.catalog.CatalogLoader;
import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.Timestamp;
import org.apache.paimon.manifest.PartitionEntry;
import org.apache.paimon.operation.FileStoreScan;
import org.apache.paimon.types.DataTypeRoot;
import org.apache.paimon.types.DataType;
import org.apache.paimon.types.DataTypeChecks;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.DateTimeUtils;
import org.apache.paimon.utils.RowDataToObjectArrayConverter;

import org.apache.paimon.shade.guava30.com.google.common.primitives.UnsignedBytes;
Expand All @@ -51,11 +54,20 @@ public abstract class PartitionExpireStrategy implements Serializable {
protected final List<String> partitionKeys;
protected final String partitionDefaultName;
private final RowDataToObjectArrayConverter toObjectArrayConverter;
@Nullable private final String timestampFormatter;

public PartitionExpireStrategy(RowType partitionType, String partitionDefaultName) {
this(partitionType, partitionDefaultName, null);
}

public PartitionExpireStrategy(
RowType partitionType,
String partitionDefaultName,
@Nullable String timestampFormatter) {
this.toObjectArrayConverter = new RowDataToObjectArrayConverter(partitionType);
this.partitionKeys = partitionType.getFieldNames();
this.partitionDefaultName = partitionDefaultName;
this.timestampFormatter = timestampFormatter;
}

public Map<String, String> toPartitionString(Object[] array) {
Expand All @@ -78,18 +90,53 @@ public List<String> toPartitionValue(Object[] array) {
return list;
}

/**
* Whether {@link #convertPartition} renders the value at this index in a form meant for time
* extraction that can differ from the stored partition name.
*/
public boolean isNormalizedForTimeExtraction(int index) {
switch (toObjectArrayConverter.rowType().getTypeAt(index).getTypeRoot()) {
case DATE:
case TIMESTAMP_WITHOUT_TIME_ZONE:
return true;
default:
return false;
}
}

public Object[] convertPartition(BinaryRow partition) {
Object[] values = toObjectArrayConverter.convert(partition);
RowType partitionType = toObjectArrayConverter.rowType();
for (int index = 0; index < values.length; index++) {
if (values[index] != null
&& partitionType.getTypeAt(index).getTypeRoot() == DataTypeRoot.DATE) {
values[index] = LocalDate.ofEpochDay((Integer) values[index]);
if (values[index] == null) {
continue;
}
DataType type = partitionType.getTypeAt(index);
switch (type.getTypeRoot()) {
case DATE:
values[index] = LocalDate.ofEpochDay((Integer) values[index]);
break;
case TIMESTAMP_WITHOUT_TIME_ZONE:
values[index] =
formatTimestampForTimeExtraction((Timestamp) values[index], type);
break;
default:
break;
}
}
return values;
}

private String formatTimestampForTimeExtraction(Timestamp timestamp, DataType type) {
if (timestampFormatter != null) {
// Keep the Timestamp#toString input an explicitly configured formatter expects.
return timestamp.toString();
}
// Timestamp#toString uses the ISO 'T' separator, which the default partition time
// formatter cannot parse. Render it as the SQL timestamp string.
return DateTimeUtils.formatTimestamp(timestamp, DataTypeChecks.getPrecision(type));
}

public abstract List<PartitionEntry> selectExpiredPartitions(
FileStoreScan scan, LocalDateTime expirationTime);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ public class PartitionValuesTimeExpireStrategy extends PartitionExpireStrategy {
private final boolean retainLatest;

public PartitionValuesTimeExpireStrategy(CoreOptions options, RowType partitionType) {
super(partitionType, options.partitionDefaultName());
super(partitionType, options.partitionDefaultName(), options.partitionTimestampFormatter());
String timePattern = options.partitionTimestampPattern();
String timeFormatter = options.partitionTimestampFormatter();
this.timeExtractor = new PartitionTimeExtractor(timePattern, timeFormatter);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import org.apache.paimon.catalog.Catalog;
import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.partition.PartitionStatistics;

import java.util.List;
Expand All @@ -33,6 +34,13 @@ void createPartitions(List<Map<String, String>> partitions)

void dropPartitions(List<Map<String, String>> partitions) throws Catalog.TableNotExistException;

/** Drop partitions given by their partition rows, see {@link Catalog#dropBinaryPartitions}. */
default void dropBinaryPartitions(List<BinaryRow> partitions)
throws Catalog.TableNotExistException {
throw new UnsupportedOperationException(
getClass().getName() + " does not support dropping partitions by partition rows.");
}

void alterPartitions(List<PartitionStatistics> partitions)
throws Catalog.TableNotExistException;

Expand All @@ -51,6 +59,12 @@ public void dropPartitions(List<Map<String, String>> partitions)
catalog.dropPartitions(identifier, partitions);
}

@Override
public void dropBinaryPartitions(List<BinaryRow> partitions)
throws Catalog.TableNotExistException {
catalog.dropBinaryPartitions(identifier, partitions);
}

@Override
public void alterPartitions(List<PartitionStatistics> partitions)
throws Catalog.TableNotExistException {
Expand Down
Loading
Loading