diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcClient.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcClient.java
index d19d18b31a4c32..e5c3f7e7a66c33 100644
--- a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcClient.java
+++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcClient.java
@@ -26,6 +26,8 @@
import org.apache.arrow.adbc.driver.jni.JniDriver;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import java.io.Closeable;
import java.nio.file.Path;
@@ -48,9 +50,20 @@
* constructor: an FE follower replaying the edit log builds every catalog, and its filesystem layout need
* not match the leader's, so a missing driver file would otherwise stop FE from starting instead of
* failing the one catalog that cannot work.
+ *
+ *
Closing never frees what a call is still using. A catalog is dropped or altered on one thread
+ * while others may still be inside {@link #withConnection} for it -- a statistics loader, a statement being
+ * planned. Closing the {@code AdbcDatabase} closes every connection still open on it as well (the JNI driver
+ * closes its children first), and native driver state freed under a thread that is inside the driver
+ * crashes the whole FE process rather than failing that one call. So {@link #close} only refuses new calls,
+ * and the native objects are released by whoever leaves last: {@code close} itself when no call is in
+ * flight, otherwise the last call to finish. Waiting for those calls inside {@code close} instead would hold
+ * DROP and ALTER CATALOG for as long as a remote source takes to answer.
*/
public class AdbcClient implements Closeable {
+ private static final Logger LOG = LogManager.getLogger(AdbcClient.class);
+
private final Path driverPath;
private final String driverUrl;
private final String entrypoint;
@@ -59,9 +72,12 @@ public class AdbcClient implements Closeable {
private final String password;
private final Map driverOptions;
- private volatile BufferAllocator allocator;
- private volatile AdbcDatabase database;
- private volatile boolean closed;
+ // All guarded by this.
+ private BufferAllocator allocator;
+ private AdbcDatabase database;
+ private boolean closed;
+ /** Calls inside {@link #withConnection}; the last one to leave a closed client releases the driver. */
+ private int callsInFlight;
public AdbcClient(Path driverPath, String driverUrl, String entrypoint, String uri,
String user, String password, Map driverOptions) {
@@ -83,7 +99,7 @@ public AdbcClient(Path driverPath, String driverUrl, String entrypoint, String u
* cost does not show.
*/
public T withConnection(AdbcConnectionCall body) {
- AdbcDatabase db = getOrOpenDatabase();
+ AdbcDatabase db = enter();
ClassLoader previous = Thread.currentThread().getContextClassLoader();
try {
// The plugin loads child-first and the ADBC/Arrow classes live in it; pin the context
@@ -101,43 +117,60 @@ public T withConnection(AdbcConnectionCall body) {
throw new DorisConnectorException("ADBC operation failed: " + e.getMessage(), e);
} finally {
Thread.currentThread().setContextClassLoader(previous);
+ leave();
}
}
- private AdbcDatabase getOrOpenDatabase() {
+ /** Counts the calling thread in, opening the database on first use. */
+ private synchronized AdbcDatabase enter() {
if (closed) {
throw new DorisConnectorException("AdbcClient has been closed");
}
- AdbcDatabase db = database;
- if (db != null) {
- return db;
+ if (database == null) {
+ AdbcDriverPathResolver.checkExists(driverPath, driverUrl);
+ ClassLoader previous = Thread.currentThread().getContextClassLoader();
+ try {
+ Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
+ if (allocator == null) {
+ allocator = new RootAllocator();
+ }
+ database = openDatabase(allocator, buildParameters());
+ } catch (AdbcException e) {
+ throw translate(e, "Failed to open the ADBC driver " + driverPath);
+ } catch (UnsatisfiedLinkError e) {
+ throw new DorisConnectorException("Failed to load the ADBC JNI bridge."
+ + " FE loads it from the directory named by the JVM property"
+ + " arrow.adbc.driver.jni.library.path (set in fe.conf, normally"
+ + " ${DORIS_HOME}/lib): " + e.getMessage(), e);
+ } finally {
+ Thread.currentThread().setContextClassLoader(previous);
+ }
}
+ callsInFlight++;
+ return database;
+ }
+
+ /** Counts the calling thread out; the last call to leave a closed client releases the driver. */
+ private void leave() {
synchronized (this) {
- if (closed) {
- throw new DorisConnectorException("AdbcClient has been closed");
+ callsInFlight--;
+ if (!closed || callsInFlight > 0) {
+ return;
}
- if (database == null) {
- AdbcDriverPathResolver.checkExists(driverPath, driverUrl);
- ClassLoader previous = Thread.currentThread().getContextClassLoader();
- try {
- Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- if (allocator == null) {
- allocator = new RootAllocator();
- }
- database = new JniDriver(allocator).open(buildParameters());
- } catch (AdbcException e) {
- throw translate(e, "Failed to open the ADBC driver " + driverPath);
- } catch (UnsatisfiedLinkError e) {
- throw new DorisConnectorException("Failed to load the ADBC JNI bridge."
- + " FE loads it from the directory named by the JVM property"
- + " arrow.adbc.driver.jni.library.path (set in fe.conf, normally"
- + " ${DORIS_HOME}/lib): " + e.getMessage(), e);
- } finally {
- Thread.currentThread().setContextClassLoader(previous);
- }
- }
- return database;
}
+ // close() found this call still inside the driver and left the release to it. A failure is logged, as
+ // the catalog logs a failed close (PluginDrivenExternalCatalog#closeResources), not thrown: the call
+ // itself is done, and its caller is not the one that closed the catalog.
+ try {
+ release();
+ } catch (RuntimeException e) {
+ LOG.warn("Failed to release the ADBC driver {} of a closed catalog", driverPath, e);
+ }
+ }
+
+ /** The one place the database is created, so a test can stand a recording one in for the driver. */
+ AdbcDatabase openDatabase(BufferAllocator allocator, Map parameters) throws AdbcException {
+ return new JniDriver(allocator).open(parameters);
}
private Map buildParameters() {
@@ -184,20 +217,40 @@ static DorisConnectorException translate(AdbcException e, String context) {
return new DorisConnectorException(sb.toString(), e);
}
+ /**
+ * Refuses new calls, and releases the driver now unless a call is still inside it -- then the last such
+ * call releases it as it leaves. Does not wait for those calls; see the class comment.
+ */
@Override
- public synchronized void close() {
- closed = true;
+ public void close() {
+ synchronized (this) {
+ closed = true;
+ if (callsInFlight > 0) {
+ return;
+ }
+ }
+ release();
+ }
+
+ /** Frees the database and then its allocator; a client never used, or already released, holds neither. */
+ private void release() {
+ AdbcDatabase db;
+ BufferAllocator alloc;
+ synchronized (this) {
+ db = database;
+ alloc = allocator;
+ database = null;
+ allocator = null;
+ }
try {
- if (database != null) {
- database.close();
+ if (db != null) {
+ db.close();
}
} catch (Exception e) {
throw new DorisConnectorException("Failed to close the ADBC database: " + e.getMessage(), e);
} finally {
- database = null;
- if (allocator != null) {
- allocator.close();
- allocator = null;
+ if (alloc != null) {
+ alloc.close();
}
}
}
diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcClientTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcClientTest.java
index 05583d941920a6..f87978c97c08f4 100644
--- a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcClientTest.java
+++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcClientTest.java
@@ -19,18 +19,35 @@
import org.apache.doris.connector.spi.DorisConnectorException;
+import org.apache.arrow.adbc.core.AdbcConnection;
+import org.apache.arrow.adbc.core.AdbcDatabase;
import org.apache.arrow.adbc.core.AdbcException;
+import org.apache.arrow.adbc.core.AdbcStatement;
import org.apache.arrow.adbc.core.AdbcStatusCode;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.ipc.ArrowReader;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
import java.nio.file.Path;
+import java.util.List;
import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
/**
* Exercises {@link AdbcClient} against the real JNI bridge and the real SQLite driver from thirdparty.
- * Skips (loudly) when those libraries are absent -- see {@link AdbcNativeTestSupport}.
+ * Skips (loudly) when those libraries are absent -- see {@link AdbcNativeTestSupport}. The tests of when the
+ * driver is released stand a {@link RecordingClient} in for it instead, so they run everywhere.
*/
class AdbcClientTest {
@@ -74,6 +91,135 @@ void usingAClosedClientFailsLoud(@TempDir Path tempDir) {
Assertions.assertTrue(e.getMessage().contains("closed"), e.getMessage());
}
+ @Test
+ void closeLeavesACallThatIsStillUsingTheDriverAlone(@TempDir Path tempDir) throws Exception {
+ AdbcClient client = sqliteClient(tempDir.resolve("probe.db"));
+ CountDownLatch insideCall = new CountDownLatch(1);
+ CountDownLatch mayContinue = new CountDownLatch(1);
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ try {
+ Future call = caller.submit(() -> client.withConnection(connection -> {
+ insideCall.countDown();
+ Assertions.assertTrue(mayContinue.await(30, TimeUnit.SECONDS));
+ // Back into the driver after the client was closed, as a statistics loader is when the catalog
+ // it reads is dropped under it. Closing the database used to close this connection too; here
+ // the JNI handle check turns that into an error, but a thread already inside the driver read
+ // freed memory and took FE down with it (SIGSEGV in sqlite3FindTable).
+ return connection.getCurrentCatalog();
+ }));
+ Assertions.assertTrue(insideCall.await(30, TimeUnit.SECONDS));
+ closeWithoutWaitingForCalls(client);
+ mayContinue.countDown();
+ Assertions.assertEquals("main", call.get(30, TimeUnit.SECONDS));
+ } finally {
+ caller.shutdownNow();
+ }
+ }
+
+ @Test
+ void closeLeavesAResultACallIsStillReadingAlone(@TempDir Path tempDir) throws Exception {
+ RecordingClient client = new RecordingClient(Files.createFile(tempDir.resolve("driver.so")));
+ CountDownLatch insideCall = new CountDownLatch(1);
+ CountDownLatch mayContinue = new CountDownLatch(1);
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ try {
+ Future> call = caller.submit(() -> client.withConnection(connection -> {
+ try (ArrowReader reader = connection.getObjects(
+ AdbcConnection.GetObjectsDepth.DB_SCHEMAS, null, null, null, null, null)) {
+ insideCall.countDown();
+ Assertions.assertTrue(mayContinue.await(30, TimeUnit.SECONDS));
+ // The getObjects result behind getTableHandle, read after the client was closed, as a
+ // statistics load reads it when the catalog is dropped under it. Closing the client used to
+ // release the database, and with it this connection and its result, and then the allocator the
+ // result lives in; with the SQLite driver, a read caught in between took FE down (SIGSEGV in
+ // get_next_wrapper or PrivateArrowSchemaDeepCopy).
+ return AdbcObjectsReader.readNamespaces(reader);
+ }
+ }));
+ Assertions.assertTrue(insideCall.await(30, TimeUnit.SECONDS));
+ closeWithoutWaitingForCalls(client);
+ mayContinue.countDown();
+ Assertions.assertEquals(List.of(new AdbcNamespace("main", "")), call.get(30, TimeUnit.SECONDS));
+ } finally {
+ caller.shutdownNow();
+ }
+ }
+
+ @Test
+ void theLastCallToLeaveAClosedClientReleasesTheDatabase(@TempDir Path tempDir) throws Exception {
+ RecordingClient client = new RecordingClient(Files.createFile(tempDir.resolve("driver.so")));
+ CountDownLatch insideCall = new CountDownLatch(1);
+ CountDownLatch mayLeave = new CountDownLatch(1);
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ try {
+ Future call = caller.submit(() -> client.withConnection(connection -> {
+ insideCall.countDown();
+ Assertions.assertTrue(mayLeave.await(30, TimeUnit.SECONDS));
+ return "done";
+ }));
+ Assertions.assertTrue(insideCall.await(30, TimeUnit.SECONDS));
+
+ closeWithoutWaitingForCalls(client);
+ // Refused from now on, yet nothing is released while a call is still inside.
+ DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class,
+ () -> client.withConnection(connection -> "late"));
+ Assertions.assertTrue(e.getMessage().contains("closed"), e.getMessage());
+ Assertions.assertEquals(0, client.database.closes.get());
+
+ mayLeave.countDown();
+ Assertions.assertEquals("done", call.get(30, TimeUnit.SECONDS));
+ Assertions.assertEquals(1, client.database.closes.get());
+ } finally {
+ caller.shutdownNow();
+ }
+ client.close();
+ Assertions.assertEquals(1, client.database.closes.get());
+ }
+
+ @Test
+ void releaseFailingAsTheLastCallLeavesIsNotThatCallsFailure(@TempDir Path tempDir) throws Exception {
+ RecordingClient client = new RecordingClient(Files.createFile(tempDir.resolve("driver.so")),
+ new RecordingDatabase(true));
+ CountDownLatch insideCall = new CountDownLatch(1);
+ CountDownLatch mayLeave = new CountDownLatch(1);
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ try {
+ Future call = caller.submit(() -> client.withConnection(connection -> {
+ insideCall.countDown();
+ Assertions.assertTrue(mayLeave.await(30, TimeUnit.SECONDS));
+ return "done";
+ }));
+ Assertions.assertTrue(insideCall.await(30, TimeUnit.SECONDS));
+ closeWithoutWaitingForCalls(client);
+
+ mayLeave.countDown();
+ // The call did its work; the driver failing to close as it leaves is logged, not thrown at a caller that
+ // did not close the catalog - a statistics load or a statement that happened to be the last one in.
+ Assertions.assertEquals("done", call.get(30, TimeUnit.SECONDS));
+ Assertions.assertEquals(1, client.database.closes.get());
+ } finally {
+ caller.shutdownNow();
+ }
+ // Released, if unsuccessfully: nothing is closed again.
+ client.close();
+ Assertions.assertEquals(1, client.database.closes.get());
+ }
+
+ @Test
+ void closingAClientNoCallIsUsingReleasesTheDatabaseAtOnce(@TempDir Path tempDir) throws Exception {
+ RecordingClient client = new RecordingClient(Files.createFile(tempDir.resolve("driver.so")));
+ Assertions.assertEquals("done", client.withConnection(connection -> "done"));
+ Assertions.assertEquals(0, client.database.closes.get());
+
+ client.close();
+ Assertions.assertEquals(1, client.database.closes.get());
+ }
+
+ /** Fails instead of hanging the run if close() ever starts waiting for the calls still in flight. */
+ private static void closeWithoutWaitingForCalls(AdbcClient client) throws Exception {
+ CompletableFuture.runAsync(client::close).get(30, TimeUnit.SECONDS);
+ }
+
@Test
void missingDriverFileFailsAtFirstUseNotAtConstruction(@TempDir Path tempDir) {
// An FE follower replaying the edit log constructs every catalog; if construction reached the
@@ -130,4 +276,85 @@ void meaningfulDriverMessagesAreKept() {
Assertions.assertTrue(message.contains("42P01"), message);
Assertions.assertTrue(message.contains("7"), message);
}
+
+ /** An {@link AdbcClient} whose database records how it is used instead of reaching a driver. */
+ private static final class RecordingClient extends AdbcClient {
+
+ private final RecordingDatabase database;
+
+ private RecordingClient(Path driver) {
+ this(driver, new RecordingDatabase(false));
+ }
+
+ private RecordingClient(Path driver, RecordingDatabase database) {
+ super(driver, driver.toString(), null, "file:/unused.db", null, null, Map.of());
+ this.database = database;
+ }
+
+ @Override
+ AdbcDatabase openDatabase(BufferAllocator allocator, Map parameters) {
+ database.allocator = allocator;
+ return database;
+ }
+ }
+
+ private static final class RecordingDatabase implements AdbcDatabase {
+
+ private final AtomicInteger openConnections = new AtomicInteger();
+ private final AtomicInteger closes = new AtomicInteger();
+ private final boolean closeFails;
+ /** The client's allocator, handed over when the client opens its database, as the JNI driver is given it. */
+ private BufferAllocator allocator;
+
+ private RecordingDatabase(boolean closeFails) {
+ this.closeFails = closeFails;
+ }
+
+ @Override
+ public AdbcConnection connect() {
+ openConnections.incrementAndGet();
+ return new AdbcConnection() {
+ @Override
+ public AdbcStatement createStatement() {
+ throw new AssertionError("these tests run no statement");
+ }
+
+ @Override
+ public ArrowReader getInfo(int[] infoCodes) {
+ throw new AssertionError("these tests ask for no info");
+ }
+
+ @Override
+ public ArrowReader getObjects(AdbcConnection.GetObjectsDepth depth, String catalogPattern,
+ String dbSchemaPattern, String tableNamePattern, String[] tableTypes,
+ String columnNamePattern) {
+ // SQLite's answer, in the client's allocator, which is where a driver's result lives. Not read
+ // from the SQLite driver itself: reading a driver's result into Arrow needs arrow-c-data's JNI
+ // shim, which FE UT cannot load on every host -- see AdbcNativeTestSupport.
+ try {
+ return AdbcObjectsReaderTest.response(allocator,
+ AdbcObjectsReaderTest.catalog("main", AdbcObjectsReaderTest.schema("", null)));
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ @Override
+ public void close() {
+ openConnections.decrementAndGet();
+ }
+ };
+ }
+
+ @Override
+ public void close() throws AdbcException {
+ // The JNI driver closes every connection still open on the database first, from this thread --
+ // whichever thread is still using one.
+ Assertions.assertEquals(0, openConnections.get(), "the database was released under an open connection");
+ closes.incrementAndGet();
+ if (closeFails) {
+ throw new AdbcException("the driver failed to close the database", null, AdbcStatusCode.IO, null, 0);
+ }
+ }
+ }
}
diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcObjectsReaderTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcObjectsReaderTest.java
index 92da09f5f62ea8..1275c1f597ee99 100644
--- a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcObjectsReaderTest.java
+++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcObjectsReaderTest.java
@@ -37,6 +37,7 @@
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
+import java.io.IOException;
import java.nio.channels.Channels;
import java.nio.charset.StandardCharsets;
import java.util.List;
@@ -232,12 +233,12 @@ void resultWithoutTheStandardColumnsIsRejectedByName() throws Exception {
// ========= fixture =========
/** One catalog row: the name the source reported, and its schema entries (null at depth CATALOGS). */
- private static Object[] catalog(String name, Object[]... schemas) {
+ static Object[] catalog(String name, Object[]... schemas) {
return new Object[] {name, schemas == null ? null : List.of(schemas)};
}
/** One schema entry: the name the source reported (SQLite's is the empty string) and its tables. */
- private static Object[] schema(String name, Object[]... tables) {
+ static Object[] schema(String name, Object[]... tables) {
return new Object[] {name, tables == null ? null : List.of(tables)};
}
@@ -249,9 +250,9 @@ private static Object[] table(String name, String type) {
/**
* Builds the getObjects response these rows describe and returns it as the reader a driver would have
* handed over -- written to an IPC stream and read back, so the reader sees real Arrow vectors rather
- * than the fixture's own objects.
+ * than the fixture's own objects. {@link AdbcClientTest} hands it out as a driver's {@code getObjects} result.
*/
- private static ArrowReader response(BufferAllocator allocator, Object[]... rows) throws Exception {
+ static ArrowReader response(BufferAllocator allocator, Object[]... rows) throws IOException {
VectorSchemaRoot root = VectorSchemaRoot.create(GET_OBJECTS_SCHEMA, allocator);
VarCharVector catalogs = (VarCharVector) root.getVector(CATALOG_NAME);
ListVector schemas = (ListVector) root.getVector(CATALOG_DB_SCHEMAS);