Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/generated/core_configuration.html
Original file line number Diff line number Diff line change
Expand Up @@ -1432,7 +1432,7 @@
<td><h5>partition.timestamp-pattern</h5></td>
<td style="word-wrap: break-word;">(none)</td>
<td>String</td>
<td>You can specify a pattern to get a timestamp from partitions. The formatter pattern is defined by 'partition.timestamp-formatter'.<ul><li>By default, read from the first field.</li><li>If the timestamp in the partition is a single field called 'dt', you can use '$dt'.</li><li>If it is spread across multiple fields for year, month, day, and hour, you can use '$year-$month-$day $hour:00:00'.</li><li>If the timestamp is in fields dt and hour, you can use '$dt $hour:00:00'.</li></ul></td>
<td>You can specify a pattern to get a timestamp from partitions. The formatter pattern is defined by 'partition.timestamp-formatter'.<ul><li>By default, read from the first field.</li><li>If the timestamp in the partition is a single field called 'dt', you can use '$dt'.</li><li>If it is spread across multiple fields for year, month, day, and hour, you can use '$year-$month-$day $hour:00:00'.</li><li>If the timestamp is in fields dt and hour, you can use '$dt $hour:00:00'.</li><li>If dt is a DATE field, whose value is the number of days since 1970-01-01, you can use '$dt::epoch_day $hour:00:00'. '::epoch_day' turns the number into yyyy-MM-dd.</li></ul></td>
</tr>
<tr>
<td><h5>pk-clustering-override</h5></td>
Expand Down
6 changes: 5 additions & 1 deletion paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
Original file line number Diff line number Diff line change
Expand Up @@ -1566,7 +1566,11 @@ public String toString() {
+ " you can use '$year-$month-$day $hour:00:00'."),
text(
"If the timestamp is in fields dt and hour, you can use '$dt "
+ "$hour:00:00'."))
+ "$hour:00:00'."),
text(
"If dt is a DATE field, whose value is the number of days since "
+ "1970-01-01, you can use '$dt::epoch_day $hour:00:00'. "
+ "'::epoch_day' turns the number into yyyy-MM-dd."))
.build());

public static final ConfigOption<Boolean> PARTITION_TIMESTAMP_FORMAT_STRICT =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.regex.Pattern;

import static java.time.temporal.ChronoField.DAY_OF_MONTH;
import static java.time.temporal.ChronoField.HOUR_OF_DAY;
Expand All @@ -48,6 +49,16 @@ public class PartitionTimeExtractor implements Serializable {

private static final long serialVersionUID = 1L;

/**
* Cast marker after a variable of the pattern, e.g. {@code $dt::epoch_day $hour:00:00}. The
* value of a DATE partition column is the number of days since 1970-01-01, which no {@link
* DateTimeFormatter} can parse, so such a value is converted to {@code yyyy-MM-dd} first. A
* value that is not a whole number, such as a formatted date, is used as it is.
*/
public static final String EPOCH_DAY_CAST = "::epoch_day";

private static final Pattern EPOCH_DAY = Pattern.compile("-?\\d{1,9}");

private static final DateTimeFormatter TIMESTAMP_FORMATTER =
new DateTimeFormatterBuilder()
.appendValue(YEAR, 1, 10, SignStyle.NORMAL)
Expand Down Expand Up @@ -124,6 +135,10 @@ private static String replacePattern(
if (pattern.regionMatches(i + 1, key, 0, key.length())) {
matchedValue = partitionValues.get(idx).toString();
i += 1 + key.length();
if (pattern.startsWith(EPOCH_DAY_CAST, i)) {
matchedValue = fromEpochDay(matchedValue);
i += EPOCH_DAY_CAST.length();
}
break;
}
}
Expand All @@ -138,6 +153,13 @@ private static String replacePattern(
return builder.toString();
}

private static String fromEpochDay(String value) {
if (!EPOCH_DAY.matcher(value).matches()) {
return value;
}
return LocalDate.ofEpochDay(Long.parseLong(value)).toString();
}

private static LocalDateTime toLocalDateTime(
String timestampString, @Nullable String formatterPattern) {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -127,4 +127,66 @@ public void testExtractNonDateFormattedPartition() {
DateTimeParseException.class,
"Text 'unknown' could not be parsed at index 0"));
}

@Test
public void testEpochDayCastWithHourAndFormatter() {
PartitionTimeExtractor extractor =
new PartitionTimeExtractor("$pd::epoch_day $ph:00:00", "yyyy-MM-dd H:mm:ss");
assertThat(extractor.extract(Arrays.asList("pd", "ph"), Arrays.asList(20477, 0)))
.isEqualTo(LocalDateTime.parse("2026-01-24T00:00:00"));
assertThat(extractor.extract(Arrays.asList("pd", "ph"), Arrays.asList(20477L, "00")))
.isEqualTo(LocalDateTime.parse("2026-01-24T00:00:00"));
assertThat(extractor.extract(Arrays.asList("pd", "ph"), Arrays.asList(20478, 23)))
.isEqualTo(LocalDateTime.parse("2026-01-25T23:00:00"));
}

@Test
public void testEpochDayCastWithHourWithoutFormatter() {
PartitionTimeExtractor extractor =
new PartitionTimeExtractor("$pd::epoch_day $ph:00:00", null);
assertThat(extractor.extract(Arrays.asList("pd", "ph"), Arrays.asList(20477, 0)))
.isEqualTo(LocalDateTime.parse("2026-01-24T00:00:00"));
assertThat(extractor.extract(Arrays.asList("pd", "ph"), Arrays.asList(20479, 1)))
.isEqualTo(LocalDateTime.parse("2026-01-26T01:00:00"));
}

@Test
public void testEpochDayCastAlone() {
PartitionTimeExtractor extractor = new PartitionTimeExtractor("$pd::epoch_day", null);
assertThat(
extractor.extract(
Collections.singletonList("pd"), Collections.singletonList(20477)))
.isEqualTo(LocalDateTime.parse("2026-01-24T00:00:00"));
assertThat(
extractor.extract(
Collections.singletonList("pd"), Collections.singletonList(-1)))
.isEqualTo(LocalDateTime.parse("1969-12-31T00:00:00"));
}

@Test
public void testEpochDayCastKeepsFormattedDate() {
// A DATE value read from a partition path is already a date string.
PartitionTimeExtractor extractor =
new PartitionTimeExtractor("$pd::epoch_day $ph:00:00", null);
assertThat(extractor.extract(Arrays.asList("pd", "ph"), Arrays.asList("2026-01-24", "5")))
.isEqualTo(LocalDateTime.parse("2026-01-24T05:00:00"));
}

@Test
public void testEpochDayCastWithPrefixVariableNames() {
PartitionTimeExtractor extractor =
new PartitionTimeExtractor("$dt::epoch_day $dt1:00:00", null);
assertThat(extractor.extract(Arrays.asList("dt", "dt1"), Arrays.asList(20477, 7)))
.isEqualTo(LocalDateTime.parse("2026-01-24T07:00:00"));
}

@Test
public void testIntegerWithoutEpochDayCastIsUnchanged() {
PartitionTimeExtractor extractor = new PartitionTimeExtractor("$dt", "yyyyMMdd");
assertThat(
extractor.extract(
Collections.singletonList("dt"),
Collections.singletonList(20230115)))
.isEqualTo(LocalDateTime.parse("2023-01-15T00:00:00"));
}
}
Loading