Repository navigation
fix: warn when native and JVM timezone databases differ #6353
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -27,6 +27,7 @@ | |
| import java.io.InputStreamReader; | ||
| import java.nio.file.Files; | ||
| import java.nio.file.StandardCopyOption; | ||
| import java.time.zone.ZoneRulesProvider; | ||
| import java.util.concurrent.atomic.AtomicBoolean; | ||
|
|
||
| import org.slf4j.Logger; | ||
|
|
@@ -101,6 +102,7 @@ static synchronized void load() { | |
| } | ||
|
|
||
| initWithLogConf(); | ||
| warnOnTzdataMismatch(); | ||
| // Only set the Arrow properties when debugging mode is off | ||
| if (!(boolean) CometConf.COMET_DEBUG_ENABLED().get()) { | ||
| setArrowProperties(); | ||
|
|
@@ -177,6 +179,35 @@ private static void initWithLogConf() { | |
| init(logConfPath, logLevel); | ||
| } | ||
|
|
||
| /** | ||
| * Native code converts between instants and local time with the IANA timezone database that | ||
| * chrono-tz compiles into libcomet, while Spark uses the JVM's. When the two versions differ, | ||
| * local times in timezones whose rules changed between them can differ from Spark's. | ||
| */ | ||
| private static void warnOnTzdataMismatch() { | ||
| try { | ||
| String warning = | ||
| tzdataMismatchWarning(getTzdataVersion(), ZoneRulesProvider.getVersions("UTC").lastKey()); | ||
| if (warning != null) { | ||
| LOG.warn(warning); | ||
| } | ||
| } catch (Throwable t) { | ||
| LOG.debug("Could not compare timezone database versions", t); | ||
| } | ||
| } | ||
|
|
||
| /** The warning to log when native code and the JVM use different tzdata versions, or null. */ | ||
| static String tzdataMismatchWarning(String nativeVersion, String jvmVersion) { | ||
| if (nativeVersion.equals(jvmVersion)) { | ||
| return null; | ||
| } | ||
| return String.format( | ||
| "Comet's native library uses timezone database %s, but the JVM uses %s. Local times that " | ||
| + "Comet computes natively can differ from Spark's in timezones whose rules changed " | ||
| + "between these versions.", | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. JDK builds ship different tzdata versions. For example, the |
||
| nativeVersion, jvmVersion); | ||
| } | ||
|
|
||
| private static void cleanupOldTempLibs() { | ||
| String tempFolder = new File(System.getProperty("java.io.tmpdir")).getAbsolutePath(); | ||
| File dir = new File(tempFolder); | ||
|
|
@@ -331,4 +362,11 @@ public static void releaseNative() throws Throwable { | |
| * @return true if object_store can construct both a store and an object key for this URL | ||
| */ | ||
| public static native boolean isObjectStoreSchemeSupported(String url); | ||
|
|
||
| /** | ||
| * The version of the IANA timezone database that native code uses, such as "2025b". | ||
| * | ||
| * @return the version compiled into libcomet | ||
| */ | ||
| public static native String getTzdataVersion(); | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Nit: the
date_truncbullet at the top of this page says non-UTC sessions go through the JVM codegen dispatcher by default. So I'd expect nativedate_truncto be affected only withspark.comet.expression.TruncTimestamp.allowIncompatible=true. Also, "parsing strings as timestamps" looks covered by the casts already. Would it make sense to tighten this list?