diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcClient.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcClient.java index d19d18b31a4c32..e5c3f7e7a66c33 100644 --- a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcClient.java +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcClient.java @@ -26,6 +26,8 @@ import org.apache.arrow.adbc.driver.jni.JniDriver; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.io.Closeable; import java.nio.file.Path; @@ -48,9 +50,20 @@ * constructor: an FE follower replaying the edit log builds every catalog, and its filesystem layout need * not match the leader's, so a missing driver file would otherwise stop FE from starting instead of * failing the one catalog that cannot work. + * + *

Closing never frees what a call is still using. A catalog is dropped or altered on one thread + * while others may still be inside {@link #withConnection} for it -- a statistics loader, a statement being + * planned. Closing the {@code AdbcDatabase} closes every connection still open on it as well (the JNI driver + * closes its children first), and native driver state freed under a thread that is inside the driver + * crashes the whole FE process rather than failing that one call. So {@link #close} only refuses new calls, + * and the native objects are released by whoever leaves last: {@code close} itself when no call is in + * flight, otherwise the last call to finish. Waiting for those calls inside {@code close} instead would hold + * DROP and ALTER CATALOG for as long as a remote source takes to answer. */ public class AdbcClient implements Closeable { + private static final Logger LOG = LogManager.getLogger(AdbcClient.class); + private final Path driverPath; private final String driverUrl; private final String entrypoint; @@ -59,9 +72,12 @@ public class AdbcClient implements Closeable { private final String password; private final Map driverOptions; - private volatile BufferAllocator allocator; - private volatile AdbcDatabase database; - private volatile boolean closed; + // All guarded by this. + private BufferAllocator allocator; + private AdbcDatabase database; + private boolean closed; + /** Calls inside {@link #withConnection}; the last one to leave a closed client releases the driver. */ + private int callsInFlight; public AdbcClient(Path driverPath, String driverUrl, String entrypoint, String uri, String user, String password, Map driverOptions) { @@ -83,7 +99,7 @@ public AdbcClient(Path driverPath, String driverUrl, String entrypoint, String u * cost does not show. */ public T withConnection(AdbcConnectionCall body) { - AdbcDatabase db = getOrOpenDatabase(); + AdbcDatabase db = enter(); ClassLoader previous = Thread.currentThread().getContextClassLoader(); try { // The plugin loads child-first and the ADBC/Arrow classes live in it; pin the context @@ -101,43 +117,60 @@ public T withConnection(AdbcConnectionCall body) { throw new DorisConnectorException("ADBC operation failed: " + e.getMessage(), e); } finally { Thread.currentThread().setContextClassLoader(previous); + leave(); } } - private AdbcDatabase getOrOpenDatabase() { + /** Counts the calling thread in, opening the database on first use. */ + private synchronized AdbcDatabase enter() { if (closed) { throw new DorisConnectorException("AdbcClient has been closed"); } - AdbcDatabase db = database; - if (db != null) { - return db; + if (database == null) { + AdbcDriverPathResolver.checkExists(driverPath, driverUrl); + ClassLoader previous = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); + if (allocator == null) { + allocator = new RootAllocator(); + } + database = openDatabase(allocator, buildParameters()); + } catch (AdbcException e) { + throw translate(e, "Failed to open the ADBC driver " + driverPath); + } catch (UnsatisfiedLinkError e) { + throw new DorisConnectorException("Failed to load the ADBC JNI bridge." + + " FE loads it from the directory named by the JVM property" + + " arrow.adbc.driver.jni.library.path (set in fe.conf, normally" + + " ${DORIS_HOME}/lib): " + e.getMessage(), e); + } finally { + Thread.currentThread().setContextClassLoader(previous); + } } + callsInFlight++; + return database; + } + + /** Counts the calling thread out; the last call to leave a closed client releases the driver. */ + private void leave() { synchronized (this) { - if (closed) { - throw new DorisConnectorException("AdbcClient has been closed"); + callsInFlight--; + if (!closed || callsInFlight > 0) { + return; } - if (database == null) { - AdbcDriverPathResolver.checkExists(driverPath, driverUrl); - ClassLoader previous = Thread.currentThread().getContextClassLoader(); - try { - Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); - if (allocator == null) { - allocator = new RootAllocator(); - } - database = new JniDriver(allocator).open(buildParameters()); - } catch (AdbcException e) { - throw translate(e, "Failed to open the ADBC driver " + driverPath); - } catch (UnsatisfiedLinkError e) { - throw new DorisConnectorException("Failed to load the ADBC JNI bridge." - + " FE loads it from the directory named by the JVM property" - + " arrow.adbc.driver.jni.library.path (set in fe.conf, normally" - + " ${DORIS_HOME}/lib): " + e.getMessage(), e); - } finally { - Thread.currentThread().setContextClassLoader(previous); - } - } - return database; } + // close() found this call still inside the driver and left the release to it. A failure is logged, as + // the catalog logs a failed close (PluginDrivenExternalCatalog#closeResources), not thrown: the call + // itself is done, and its caller is not the one that closed the catalog. + try { + release(); + } catch (RuntimeException e) { + LOG.warn("Failed to release the ADBC driver {} of a closed catalog", driverPath, e); + } + } + + /** The one place the database is created, so a test can stand a recording one in for the driver. */ + AdbcDatabase openDatabase(BufferAllocator allocator, Map parameters) throws AdbcException { + return new JniDriver(allocator).open(parameters); } private Map buildParameters() { @@ -184,20 +217,40 @@ static DorisConnectorException translate(AdbcException e, String context) { return new DorisConnectorException(sb.toString(), e); } + /** + * Refuses new calls, and releases the driver now unless a call is still inside it -- then the last such + * call releases it as it leaves. Does not wait for those calls; see the class comment. + */ @Override - public synchronized void close() { - closed = true; + public void close() { + synchronized (this) { + closed = true; + if (callsInFlight > 0) { + return; + } + } + release(); + } + + /** Frees the database and then its allocator; a client never used, or already released, holds neither. */ + private void release() { + AdbcDatabase db; + BufferAllocator alloc; + synchronized (this) { + db = database; + alloc = allocator; + database = null; + allocator = null; + } try { - if (database != null) { - database.close(); + if (db != null) { + db.close(); } } catch (Exception e) { throw new DorisConnectorException("Failed to close the ADBC database: " + e.getMessage(), e); } finally { - database = null; - if (allocator != null) { - allocator.close(); - allocator = null; + if (alloc != null) { + alloc.close(); } } } diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcClientTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcClientTest.java index 05583d941920a6..f87978c97c08f4 100644 --- a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcClientTest.java +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcClientTest.java @@ -19,18 +19,35 @@ import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.arrow.adbc.core.AdbcConnection; +import org.apache.arrow.adbc.core.AdbcDatabase; import org.apache.arrow.adbc.core.AdbcException; +import org.apache.arrow.adbc.core.AdbcStatement; import org.apache.arrow.adbc.core.AdbcStatusCode; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.ipc.ArrowReader; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; /** * Exercises {@link AdbcClient} against the real JNI bridge and the real SQLite driver from thirdparty. - * Skips (loudly) when those libraries are absent -- see {@link AdbcNativeTestSupport}. + * Skips (loudly) when those libraries are absent -- see {@link AdbcNativeTestSupport}. The tests of when the + * driver is released stand a {@link RecordingClient} in for it instead, so they run everywhere. */ class AdbcClientTest { @@ -74,6 +91,135 @@ void usingAClosedClientFailsLoud(@TempDir Path tempDir) { Assertions.assertTrue(e.getMessage().contains("closed"), e.getMessage()); } + @Test + void closeLeavesACallThatIsStillUsingTheDriverAlone(@TempDir Path tempDir) throws Exception { + AdbcClient client = sqliteClient(tempDir.resolve("probe.db")); + CountDownLatch insideCall = new CountDownLatch(1); + CountDownLatch mayContinue = new CountDownLatch(1); + ExecutorService caller = Executors.newSingleThreadExecutor(); + try { + Future call = caller.submit(() -> client.withConnection(connection -> { + insideCall.countDown(); + Assertions.assertTrue(mayContinue.await(30, TimeUnit.SECONDS)); + // Back into the driver after the client was closed, as a statistics loader is when the catalog + // it reads is dropped under it. Closing the database used to close this connection too; here + // the JNI handle check turns that into an error, but a thread already inside the driver read + // freed memory and took FE down with it (SIGSEGV in sqlite3FindTable). + return connection.getCurrentCatalog(); + })); + Assertions.assertTrue(insideCall.await(30, TimeUnit.SECONDS)); + closeWithoutWaitingForCalls(client); + mayContinue.countDown(); + Assertions.assertEquals("main", call.get(30, TimeUnit.SECONDS)); + } finally { + caller.shutdownNow(); + } + } + + @Test + void closeLeavesAResultACallIsStillReadingAlone(@TempDir Path tempDir) throws Exception { + RecordingClient client = new RecordingClient(Files.createFile(tempDir.resolve("driver.so"))); + CountDownLatch insideCall = new CountDownLatch(1); + CountDownLatch mayContinue = new CountDownLatch(1); + ExecutorService caller = Executors.newSingleThreadExecutor(); + try { + Future> call = caller.submit(() -> client.withConnection(connection -> { + try (ArrowReader reader = connection.getObjects( + AdbcConnection.GetObjectsDepth.DB_SCHEMAS, null, null, null, null, null)) { + insideCall.countDown(); + Assertions.assertTrue(mayContinue.await(30, TimeUnit.SECONDS)); + // The getObjects result behind getTableHandle, read after the client was closed, as a + // statistics load reads it when the catalog is dropped under it. Closing the client used to + // release the database, and with it this connection and its result, and then the allocator the + // result lives in; with the SQLite driver, a read caught in between took FE down (SIGSEGV in + // get_next_wrapper or PrivateArrowSchemaDeepCopy). + return AdbcObjectsReader.readNamespaces(reader); + } + })); + Assertions.assertTrue(insideCall.await(30, TimeUnit.SECONDS)); + closeWithoutWaitingForCalls(client); + mayContinue.countDown(); + Assertions.assertEquals(List.of(new AdbcNamespace("main", "")), call.get(30, TimeUnit.SECONDS)); + } finally { + caller.shutdownNow(); + } + } + + @Test + void theLastCallToLeaveAClosedClientReleasesTheDatabase(@TempDir Path tempDir) throws Exception { + RecordingClient client = new RecordingClient(Files.createFile(tempDir.resolve("driver.so"))); + CountDownLatch insideCall = new CountDownLatch(1); + CountDownLatch mayLeave = new CountDownLatch(1); + ExecutorService caller = Executors.newSingleThreadExecutor(); + try { + Future call = caller.submit(() -> client.withConnection(connection -> { + insideCall.countDown(); + Assertions.assertTrue(mayLeave.await(30, TimeUnit.SECONDS)); + return "done"; + })); + Assertions.assertTrue(insideCall.await(30, TimeUnit.SECONDS)); + + closeWithoutWaitingForCalls(client); + // Refused from now on, yet nothing is released while a call is still inside. + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> client.withConnection(connection -> "late")); + Assertions.assertTrue(e.getMessage().contains("closed"), e.getMessage()); + Assertions.assertEquals(0, client.database.closes.get()); + + mayLeave.countDown(); + Assertions.assertEquals("done", call.get(30, TimeUnit.SECONDS)); + Assertions.assertEquals(1, client.database.closes.get()); + } finally { + caller.shutdownNow(); + } + client.close(); + Assertions.assertEquals(1, client.database.closes.get()); + } + + @Test + void releaseFailingAsTheLastCallLeavesIsNotThatCallsFailure(@TempDir Path tempDir) throws Exception { + RecordingClient client = new RecordingClient(Files.createFile(tempDir.resolve("driver.so")), + new RecordingDatabase(true)); + CountDownLatch insideCall = new CountDownLatch(1); + CountDownLatch mayLeave = new CountDownLatch(1); + ExecutorService caller = Executors.newSingleThreadExecutor(); + try { + Future call = caller.submit(() -> client.withConnection(connection -> { + insideCall.countDown(); + Assertions.assertTrue(mayLeave.await(30, TimeUnit.SECONDS)); + return "done"; + })); + Assertions.assertTrue(insideCall.await(30, TimeUnit.SECONDS)); + closeWithoutWaitingForCalls(client); + + mayLeave.countDown(); + // The call did its work; the driver failing to close as it leaves is logged, not thrown at a caller that + // did not close the catalog - a statistics load or a statement that happened to be the last one in. + Assertions.assertEquals("done", call.get(30, TimeUnit.SECONDS)); + Assertions.assertEquals(1, client.database.closes.get()); + } finally { + caller.shutdownNow(); + } + // Released, if unsuccessfully: nothing is closed again. + client.close(); + Assertions.assertEquals(1, client.database.closes.get()); + } + + @Test + void closingAClientNoCallIsUsingReleasesTheDatabaseAtOnce(@TempDir Path tempDir) throws Exception { + RecordingClient client = new RecordingClient(Files.createFile(tempDir.resolve("driver.so"))); + Assertions.assertEquals("done", client.withConnection(connection -> "done")); + Assertions.assertEquals(0, client.database.closes.get()); + + client.close(); + Assertions.assertEquals(1, client.database.closes.get()); + } + + /** Fails instead of hanging the run if close() ever starts waiting for the calls still in flight. */ + private static void closeWithoutWaitingForCalls(AdbcClient client) throws Exception { + CompletableFuture.runAsync(client::close).get(30, TimeUnit.SECONDS); + } + @Test void missingDriverFileFailsAtFirstUseNotAtConstruction(@TempDir Path tempDir) { // An FE follower replaying the edit log constructs every catalog; if construction reached the @@ -130,4 +276,85 @@ void meaningfulDriverMessagesAreKept() { Assertions.assertTrue(message.contains("42P01"), message); Assertions.assertTrue(message.contains("7"), message); } + + /** An {@link AdbcClient} whose database records how it is used instead of reaching a driver. */ + private static final class RecordingClient extends AdbcClient { + + private final RecordingDatabase database; + + private RecordingClient(Path driver) { + this(driver, new RecordingDatabase(false)); + } + + private RecordingClient(Path driver, RecordingDatabase database) { + super(driver, driver.toString(), null, "file:/unused.db", null, null, Map.of()); + this.database = database; + } + + @Override + AdbcDatabase openDatabase(BufferAllocator allocator, Map parameters) { + database.allocator = allocator; + return database; + } + } + + private static final class RecordingDatabase implements AdbcDatabase { + + private final AtomicInteger openConnections = new AtomicInteger(); + private final AtomicInteger closes = new AtomicInteger(); + private final boolean closeFails; + /** The client's allocator, handed over when the client opens its database, as the JNI driver is given it. */ + private BufferAllocator allocator; + + private RecordingDatabase(boolean closeFails) { + this.closeFails = closeFails; + } + + @Override + public AdbcConnection connect() { + openConnections.incrementAndGet(); + return new AdbcConnection() { + @Override + public AdbcStatement createStatement() { + throw new AssertionError("these tests run no statement"); + } + + @Override + public ArrowReader getInfo(int[] infoCodes) { + throw new AssertionError("these tests ask for no info"); + } + + @Override + public ArrowReader getObjects(AdbcConnection.GetObjectsDepth depth, String catalogPattern, + String dbSchemaPattern, String tableNamePattern, String[] tableTypes, + String columnNamePattern) { + // SQLite's answer, in the client's allocator, which is where a driver's result lives. Not read + // from the SQLite driver itself: reading a driver's result into Arrow needs arrow-c-data's JNI + // shim, which FE UT cannot load on every host -- see AdbcNativeTestSupport. + try { + return AdbcObjectsReaderTest.response(allocator, + AdbcObjectsReaderTest.catalog("main", AdbcObjectsReaderTest.schema("", null))); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public void close() { + openConnections.decrementAndGet(); + } + }; + } + + @Override + public void close() throws AdbcException { + // The JNI driver closes every connection still open on the database first, from this thread -- + // whichever thread is still using one. + Assertions.assertEquals(0, openConnections.get(), "the database was released under an open connection"); + closes.incrementAndGet(); + if (closeFails) { + throw new AdbcException("the driver failed to close the database", null, AdbcStatusCode.IO, null, 0); + } + } + } } diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcObjectsReaderTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcObjectsReaderTest.java index 92da09f5f62ea8..1275c1f597ee99 100644 --- a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcObjectsReaderTest.java +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcObjectsReaderTest.java @@ -37,6 +37,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.nio.channels.Channels; import java.nio.charset.StandardCharsets; import java.util.List; @@ -232,12 +233,12 @@ void resultWithoutTheStandardColumnsIsRejectedByName() throws Exception { // ========= fixture ========= /** One catalog row: the name the source reported, and its schema entries (null at depth CATALOGS). */ - private static Object[] catalog(String name, Object[]... schemas) { + static Object[] catalog(String name, Object[]... schemas) { return new Object[] {name, schemas == null ? null : List.of(schemas)}; } /** One schema entry: the name the source reported (SQLite's is the empty string) and its tables. */ - private static Object[] schema(String name, Object[]... tables) { + static Object[] schema(String name, Object[]... tables) { return new Object[] {name, tables == null ? null : List.of(tables)}; } @@ -249,9 +250,9 @@ private static Object[] table(String name, String type) { /** * Builds the getObjects response these rows describe and returns it as the reader a driver would have * handed over -- written to an IPC stream and read back, so the reader sees real Arrow vectors rather - * than the fixture's own objects. + * than the fixture's own objects. {@link AdbcClientTest} hands it out as a driver's {@code getObjects} result. */ - private static ArrowReader response(BufferAllocator allocator, Object[]... rows) throws Exception { + static ArrowReader response(BufferAllocator allocator, Object[]... rows) throws IOException { VectorSchemaRoot root = VectorSchemaRoot.create(GET_OBJECTS_SCHEMA, allocator); VarCharVector catalogs = (VarCharVector) root.getVector(CATALOG_NAME); ListVector schemas = (ListVector) root.getVector(CATALOG_DB_SCHEMAS);