Skip to content
Merged
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,21 @@ public <K, V> MetaCache<K, V> create(MetaCacheDefinition<K, V> definition) {
}
}

/**
* Removes one cache registration if it is still owned by this catalog. This is intended for
* rolling back a multi-step connector construction without closing unrelated sibling caches.
*/
public void remove(MetaCache<?, ?> cache) {
MetaCache<?, ?> nonNullCache = Objects.requireNonNull(cache, "cache can not be null");
if (entries.remove(nonNullCache.name(), nonNullCache)) {
try {
nonNullCache.closeFromOwner();
} finally {
names.remove(nonNullCache.name());
}
}
}

public void invalidateCatalog() {
registry.invalidate(ScopePath.catalog());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,10 @@ public void forEach(BiConsumer<K, V> consumer) {
delegate.forEach(consumer);
}

void closeFromOwner() {
delegate.close();
}

public static final class BulkLoad<K, V> implements AutoCloseable {
private final MetaCache<K, V> owner;
private final ScopedMetaCache.BulkLoadHandle delegate;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.connector.paimon;

import org.apache.doris.connector.cache.JvmSizeUtils;
import org.apache.doris.connector.cache.MetaCacheSizeEstimate;
import org.apache.doris.connector.cache.ReflectiveObjectSizeEstimator;

import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.fs.Path;
import org.apache.paimon.privilege.PrivilegedFileStoreTable;
import org.apache.paimon.table.CatalogEnvironment;
import org.apache.paimon.table.DelegatedFileStoreTable;
import org.apache.paimon.table.FallbackReadFileStoreTable;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.FormatTable;
import org.apache.paimon.table.Table;
import org.apache.paimon.table.iceberg.IcebergTable;
import org.apache.paimon.table.lance.LanceTable;
import org.apache.paimon.table.object.ObjectTable;

import java.net.URI;
import java.util.Collections;
import java.util.IdentityHashMap;
import java.util.Set;

/**
* Retained-size formulas for Paimon table-cache entries.
*
* <p>The table's shallow size includes references to FileIO, catalog loaders, and lock factories,
* but their graphs are catalog-scoped executable services rather than entry-owned metadata. Walking
* those graphs both double-counts shared state and reaches strongly encapsulated JDK objects. The
* estimator therefore expands only immutable metadata owned by the entry.
*/
final class PaimonCacheSizeEstimator {
private PaimonCacheSizeEstimator() {
}

static MetaCacheSizeEstimate estimateTable(Identifier key, Table table, long entryOverheadBytes) {
if (table instanceof PrivilegedFileStoreTable) {
return MetaCacheSizeEstimate.incomplete(
"authorization decorators must be applied outside the metadata cache");
}
long bytes = add(entryOverheadBytes, ReflectiveObjectSizeEstimator.estimateComplete(key));
if (table instanceof FileStoreTable) {
Set<Object> visited = Collections.newSetFromMap(new IdentityHashMap<>());
bytes = add(bytes, estimateFileStoreTable((FileStoreTable) table, visited));
} else {
if (!isSupportedNonFileStoreTable(table)) {
return MetaCacheSizeEstimate.incomplete(
"unsupported retained graph for " + table.getClass().getName());
}
bytes = add(bytes, JvmSizeUtils.instanceSize(table.getClass()));
Comment thread
924060929 marked this conversation as resolved.
Resolved
bytes = add(bytes, ReflectiveObjectSizeEstimator.estimateComplete(table.rowType()));
bytes = add(bytes, ReflectiveObjectSizeEstimator.estimateComplete(table.partitionKeys()));
bytes = add(bytes, ReflectiveObjectSizeEstimator.estimateComplete(table.primaryKeys()));
bytes = add(bytes, ReflectiveObjectSizeEstimator.estimateComplete(table.options()));
bytes = add(bytes, ReflectiveObjectSizeEstimator.estimateComplete(table.comment()));
bytes = add(bytes, JvmSizeUtils.stringSize(location(table)));
}
return MetaCacheSizeEstimate.complete(bytes);
}

private static boolean isSupportedNonFileStoreTable(Table table) {
return table instanceof FormatTable
|| table instanceof ObjectTable
|| table instanceof LanceTable
|| table instanceof IcebergTable;
}

private static long estimateFileStoreTable(FileStoreTable table, Set<Object> visited) {
if (!visited.add(table)) {
return 0L;
}
long bytes = JvmSizeUtils.instanceSize(table.getClass());
if (table instanceof FallbackReadFileStoreTable) {
FallbackReadFileStoreTable fallback = (FallbackReadFileStoreTable) table;
bytes = add(bytes, estimateFileStoreTable(fallback.wrapped(), visited));
return add(bytes, estimateFileStoreTable(fallback.fallback(), visited));
}
if (table instanceof DelegatedFileStoreTable) {
return add(bytes, estimateFileStoreTable(
((DelegatedFileStoreTable) table).wrapped(), visited));
}
bytes = add(bytes, estimateCompleteOnce(table.schema(), visited));
bytes = add(bytes, estimatePath(table.location(), visited));
return add(bytes, estimateCatalogEnvironment(table.catalogEnvironment(), visited));

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Account for the lazily retained FileStore before admission

This marks a normal FileStoreTable complete while its entry-owned graph can still grow after the one-time weight reservation. In Paimon 1.3.1, both AppendOnlyFileStoreTable and PrimaryKeyFileStoreTable start with lazyStore == null; the first ordinary latestSnapshot() call from beginQuerySnapshot() invokes store() and retains a newly allocated AppendOnlyFileStore/KeyValueFileStore plus its managers and factories on the cached raw table. Under an enclosing weight limit the three explicit SDK caches are skipped, but this lazy field is still populated and ScopedMetaCache never reweighs reads, so many cached tables can exceed the configured hard byte budget. Please initialize and charge this retained graph before returning complete, reject the entry under weight governance, or cache an immutable descriptor. Extend the governed real-table test past getTable() into latestSnapshot()/newScan() to cover post-admission growth.

}

private static long estimateCompleteOnce(Object value, Set<Object> visited) {
return value == null || !visited.add(value)
? 0L : ReflectiveObjectSizeEstimator.estimateComplete(value);
}

private static long estimateCatalogEnvironment(CatalogEnvironment environment, Set<Object> visited) {
if (environment == null || !visited.add(environment)) {
return 0L;
}
long bytes = JvmSizeUtils.instanceSize(environment.getClass());
bytes = add(bytes, estimateCompleteOnce(environment.identifier(), visited));
return add(bytes, estimateStringOnce(environment.uuid(), visited));
}

private static long estimatePath(Path path, Set<Object> visited) {
if (path == null || !visited.add(path)) {
return 0L;
}
URI uri = path.toUri();
long bytes = JvmSizeUtils.instanceSize(path.getClass());
if (visited.add(uri)) {
bytes = add(bytes, JvmSizeUtils.instanceSize(uri.getClass()));
bytes = add(bytes, estimateStringOnce(uri.toString(), visited));
bytes = add(bytes, estimateStringOnce(uri.getScheme(), visited));
bytes = add(bytes, estimateStringOnce(uri.getUserInfo(), visited));
bytes = add(bytes, estimateStringOnce(uri.getHost(), visited));
bytes = add(bytes, estimateStringOnce(uri.getPath(), visited));
bytes = add(bytes, estimateStringOnce(uri.getQuery(), visited));
bytes = add(bytes, estimateStringOnce(uri.getFragment(), visited));
bytes = add(bytes, estimateStringOnce(uri.getAuthority(), visited));
bytes = add(bytes, estimateStringOnce(uri.getSchemeSpecificPart(), visited));
}
return bytes;
}

private static long estimateStringOnce(String value, Set<Object> visited) {
return value == null || !visited.add(value) ? 0L : JvmSizeUtils.stringSize(value);
}

private static String location(Table table) {
if (table instanceof FormatTable) {
return ((FormatTable) table).location();
}
if (table instanceof ObjectTable) {
return ((ObjectTable) table).location();
}
if (table instanceof LanceTable) {
return ((LanceTable) table).location();
}
if (table instanceof IcebergTable) {
return ((IcebergTable) table).location();
}
return null;
}

private static long add(long left, long right) {
return JvmSizeUtils.saturatedAdd(left, right);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -103,11 +103,19 @@ private PaimonCatalogFactory() {
* plus each flavor's {@code appendCustomCatalogOptions()}.
*/
public static Options buildCatalogOptions(PaimonCatalogProperties catalogProperties) {
return buildCatalogOptions(catalogProperties, false);
Options options = assembleCatalogOptions(catalogProperties);
// PaimonMetaCacheCatalog replaces the SDK CachingCatalog. The two cache layers cannot
// coexist because a Doris invalidation cannot evict a frozen Table from the SDK wrapper.
// Preserve the user's flag separately, but always disable the SDK wrapper itself.
options.set(CatalogOptions.CACHE_ENABLED, false);
return options;
}

static boolean isCatalogCacheEnabled(PaimonCatalogProperties catalogProperties) {
return assembleCatalogOptions(catalogProperties).get(CatalogOptions.CACHE_ENABLED);
}

static Options buildCatalogOptions(
PaimonCatalogProperties catalogProperties, boolean hasEnclosingMetaCacheWeightLimit) {
private static Options assembleCatalogOptions(PaimonCatalogProperties catalogProperties) {
Options options = new Options();
Map<String, String> props = catalogProperties.getRaw();
String flavor = catalogProperties.getFlavor();
Expand Down Expand Up @@ -141,9 +149,6 @@ static Options buildCatalogOptions(
// filesystem: nothing custom.
break;
}
if (hasEnclosingMetaCacheWeightLimit && !options.contains(CatalogOptions.CACHE_ENABLED)) {
options.set(CatalogOptions.CACHE_ENABLED, false);
}
return options;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.paimon.catalog.Catalog;
import org.apache.paimon.catalog.CatalogUtils;
import org.apache.paimon.catalog.Database;
import org.apache.paimon.catalog.DelegateCatalog;
import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.partition.Partition;
import org.apache.paimon.privilege.PrivilegedFileStoreTable;
Expand Down Expand Up @@ -304,25 +305,29 @@ public List<String> listTables(String databaseName) throws Catalog.DatabaseNotEx
public Table getTable(Identifier identifier) throws Catalog.TableNotExistException {
Table table = catalog.getTable(identifier);
Map<String, String> optionsForCopy = PaimonTableOptions.forCopy(tableOptions);
// Relation options are applied after this cached handle is returned. Defer final
// validation so a safe relation value can override an unsafe physical value.
return optionsForCopy.isEmpty() ? table : table.copy(optionsForCopy);
}

@Override
public List<Partition> listPartitions(Identifier identifier, Table table)
throws Catalog.TableNotExistException {
if (catalog instanceof RESTCatalog) {
RESTCatalog restCatalog = restCatalog(catalog);
if (restCatalog != null) {
// REST owns partition visibility when its endpoint is implemented; the bridge
// retains this effective relation copy only for the endpoint's filesystem fallback.
return PaimonRestCatalogPartitions.listPartitions(
(RESTCatalog) catalog, identifier, table);
restCatalog, identifier, table);
}
// The supplied handle already contains catalog and relation policy. Reloading by identifier
// would discard those copies before manifest enumeration reaches the final scan guard.
return CatalogUtils.listPartitionsFromFileSystem(table);
}

static RESTCatalog restCatalog(Catalog catalog) {
Catalog rootCatalog = DelegateCatalog.rootCatalog(catalog);
return rootCatalog instanceof RESTCatalog ? (RESTCatalog) rootCatalog : null;
}

@Override
public void createDatabase(String name, boolean ignoreIfExists, Map<String, String> properties)
throws Catalog.DatabaseAlreadyExistException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.paimon.catalog.CachingCatalog;
import org.apache.paimon.catalog.Catalog;
import org.apache.paimon.catalog.CatalogContext;
import org.apache.paimon.catalog.CatalogFactory;
Expand All @@ -60,7 +59,6 @@
import org.apache.paimon.hive.HiveCatalogOptions;
import org.apache.paimon.options.CatalogOptions;
import org.apache.paimon.options.Options;
import org.apache.paimon.privilege.PrivilegedCatalog;

import java.io.IOException;
import java.io.UncheckedIOException;
Expand Down Expand Up @@ -499,7 +497,7 @@ static boolean hasDlfCompatibleStorage(List<StorageProperties> storageProperties
}

Options buildCatalogOptions() {
return PaimonCatalogFactory.buildCatalogOptions(catalogProps, metaCache.hasEnclosingWeightLimit());
return PaimonCatalogFactory.buildCatalogOptions(catalogProps);
}

/**
Expand Down Expand Up @@ -543,11 +541,15 @@ private Catalog createCatalogFromContext(CatalogContext catalogContext, String f
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
return context.executeAuthenticated(() -> {
// PaimonMetaCacheCatalog installs PrivilegedCatalog after the raw metadata cache.
Catalog catalog = PaimonCatalogProperties.HMS.equals(flavor)
? createHmsCatalog(catalogContext, hmsAuth, catalogProps.getRaw(),
storageHadoopConfig)
: CatalogFactory.createCatalog(catalogContext);
return catalog;
: CatalogFactory.createUnwrappedCatalog(catalogContext, getClass().getClassLoader());
return PaimonMetaCacheCatalog.tryToCreate(catalog, metaCache,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Fence this cache-backed catalog publication against close

A statement can retain this connector while resetToUninitialized() closes it. ensureCatalog() constructs under the connector monitor, but close() does not take that monitor: it can close metaCache, observe the still-null catalog, and return after these two caches have registered but before tryToCreate returns. ensureCatalog() then publishes a wrapper backed by closed caches, so its first table/database lookup throws Scoped meta cache ... is closed, and no later close owns the raw catalog. Please add a connector lifecycle fence so a successful construction cannot publish after close (closing the losing raw/decorated catalog exactly once), plus a latch-based close-versus-construction test.

DEFAULT_TABLE_CACHE_CAPACITY, resolveTableCacheTtlSecond(catalogProps.getRaw()),
catalogContext.options(), PaimonCatalogFactory.isCatalogCacheEnabled(catalogProps),
metaCache.hasEnclosingWeightLimit());
});
} catch (Exception e) {
throw new RuntimeException(failureMessage + " (flavor=" + flavor + "): " + e.getMessage(), e);
Expand Down Expand Up @@ -579,8 +581,7 @@ static Catalog createHmsCatalog(CatalogContext catalogContext, HadoopAuthenticat
fileIO, hiveConf, clientClass, options, warehousePath.toUri().toString()));
catalog = PaimonHmsClientPool.install(catalog, hmsAuth);
catalog = PaimonHmsCatalog.install(catalog, properties, storageHadoopConfig);
catalog = CachingCatalog.tryToCreate(catalog, options);
return PrivilegedCatalog.tryToCreate(catalog, options);
return catalog;
} catch (IOException e) {
throw new UncheckedIOException(e);
}
Expand Down
Loading
Loading