Repository navigation
[refactor](paimon) Replace Paimon SDK CachingCatalog with PaimonMetaCacheCatalog #67996
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
610920b
4ff1ea5
dc52bbb
4372dc4
3005ccd
d6e5a97
9cec8c3
2b94bbb
a39998d
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,163 @@ | ||
| // Licensed to the Apache Software Foundation (ASF) under one | ||
| // or more contributor license agreements. See the NOTICE file | ||
| // distributed with this work for additional information | ||
| // regarding copyright ownership. The ASF licenses this file | ||
| // to you under the Apache License, Version 2.0 (the | ||
| // "License"); you may not use this file except in compliance | ||
| // with the License. You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, | ||
| // software distributed under the License is distributed on an | ||
| // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| // KIND, either express or implied. See the License for the | ||
| // specific language governing permissions and limitations | ||
| // under the License. | ||
|
|
||
| package org.apache.doris.connector.paimon; | ||
|
|
||
| import org.apache.doris.connector.cache.JvmSizeUtils; | ||
| import org.apache.doris.connector.cache.MetaCacheSizeEstimate; | ||
| import org.apache.doris.connector.cache.ReflectiveObjectSizeEstimator; | ||
|
|
||
| import org.apache.paimon.catalog.Identifier; | ||
| import org.apache.paimon.fs.Path; | ||
| import org.apache.paimon.privilege.PrivilegedFileStoreTable; | ||
| import org.apache.paimon.table.CatalogEnvironment; | ||
| import org.apache.paimon.table.DelegatedFileStoreTable; | ||
| import org.apache.paimon.table.FallbackReadFileStoreTable; | ||
| import org.apache.paimon.table.FileStoreTable; | ||
| import org.apache.paimon.table.FormatTable; | ||
| import org.apache.paimon.table.Table; | ||
| import org.apache.paimon.table.iceberg.IcebergTable; | ||
| import org.apache.paimon.table.lance.LanceTable; | ||
| import org.apache.paimon.table.object.ObjectTable; | ||
|
|
||
| import java.net.URI; | ||
| import java.util.Collections; | ||
| import java.util.IdentityHashMap; | ||
| import java.util.Set; | ||
|
|
||
| /** | ||
| * Retained-size formulas for Paimon table-cache entries. | ||
| * | ||
| * <p>The table's shallow size includes references to FileIO, catalog loaders, and lock factories, | ||
| * but their graphs are catalog-scoped executable services rather than entry-owned metadata. Walking | ||
| * those graphs both double-counts shared state and reaches strongly encapsulated JDK objects. The | ||
| * estimator therefore expands only immutable metadata owned by the entry. | ||
| */ | ||
| final class PaimonCacheSizeEstimator { | ||
| private PaimonCacheSizeEstimator() { | ||
| } | ||
|
|
||
| static MetaCacheSizeEstimate estimateTable(Identifier key, Table table, long entryOverheadBytes) { | ||
| if (table instanceof PrivilegedFileStoreTable) { | ||
| return MetaCacheSizeEstimate.incomplete( | ||
| "authorization decorators must be applied outside the metadata cache"); | ||
| } | ||
| long bytes = add(entryOverheadBytes, ReflectiveObjectSizeEstimator.estimateComplete(key)); | ||
| if (table instanceof FileStoreTable) { | ||
| Set<Object> visited = Collections.newSetFromMap(new IdentityHashMap<>()); | ||
| bytes = add(bytes, estimateFileStoreTable((FileStoreTable) table, visited)); | ||
| } else { | ||
| if (!isSupportedNonFileStoreTable(table)) { | ||
| return MetaCacheSizeEstimate.incomplete( | ||
| "unsupported retained graph for " + table.getClass().getName()); | ||
| } | ||
| bytes = add(bytes, JvmSizeUtils.instanceSize(table.getClass())); | ||
| bytes = add(bytes, ReflectiveObjectSizeEstimator.estimateComplete(table.rowType())); | ||
| bytes = add(bytes, ReflectiveObjectSizeEstimator.estimateComplete(table.partitionKeys())); | ||
| bytes = add(bytes, ReflectiveObjectSizeEstimator.estimateComplete(table.primaryKeys())); | ||
| bytes = add(bytes, ReflectiveObjectSizeEstimator.estimateComplete(table.options())); | ||
| bytes = add(bytes, ReflectiveObjectSizeEstimator.estimateComplete(table.comment())); | ||
| bytes = add(bytes, JvmSizeUtils.stringSize(location(table))); | ||
| } | ||
| return MetaCacheSizeEstimate.complete(bytes); | ||
| } | ||
|
|
||
| private static boolean isSupportedNonFileStoreTable(Table table) { | ||
| return table instanceof FormatTable | ||
| || table instanceof ObjectTable | ||
| || table instanceof LanceTable | ||
| || table instanceof IcebergTable; | ||
| } | ||
|
|
||
| private static long estimateFileStoreTable(FileStoreTable table, Set<Object> visited) { | ||
| if (!visited.add(table)) { | ||
| return 0L; | ||
| } | ||
| long bytes = JvmSizeUtils.instanceSize(table.getClass()); | ||
| if (table instanceof FallbackReadFileStoreTable) { | ||
| FallbackReadFileStoreTable fallback = (FallbackReadFileStoreTable) table; | ||
| bytes = add(bytes, estimateFileStoreTable(fallback.wrapped(), visited)); | ||
| return add(bytes, estimateFileStoreTable(fallback.fallback(), visited)); | ||
| } | ||
| if (table instanceof DelegatedFileStoreTable) { | ||
| return add(bytes, estimateFileStoreTable( | ||
| ((DelegatedFileStoreTable) table).wrapped(), visited)); | ||
| } | ||
| bytes = add(bytes, estimateCompleteOnce(table.schema(), visited)); | ||
| bytes = add(bytes, estimatePath(table.location(), visited)); | ||
| return add(bytes, estimateCatalogEnvironment(table.catalogEnvironment(), visited)); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [P2] Account for the lazily retained FileStore before admission This marks a normal |
||
| } | ||
|
|
||
| private static long estimateCompleteOnce(Object value, Set<Object> visited) { | ||
| return value == null || !visited.add(value) | ||
| ? 0L : ReflectiveObjectSizeEstimator.estimateComplete(value); | ||
| } | ||
|
|
||
| private static long estimateCatalogEnvironment(CatalogEnvironment environment, Set<Object> visited) { | ||
| if (environment == null || !visited.add(environment)) { | ||
| return 0L; | ||
| } | ||
| long bytes = JvmSizeUtils.instanceSize(environment.getClass()); | ||
| bytes = add(bytes, estimateCompleteOnce(environment.identifier(), visited)); | ||
| return add(bytes, estimateStringOnce(environment.uuid(), visited)); | ||
| } | ||
|
|
||
| private static long estimatePath(Path path, Set<Object> visited) { | ||
| if (path == null || !visited.add(path)) { | ||
| return 0L; | ||
| } | ||
| URI uri = path.toUri(); | ||
| long bytes = JvmSizeUtils.instanceSize(path.getClass()); | ||
| if (visited.add(uri)) { | ||
| bytes = add(bytes, JvmSizeUtils.instanceSize(uri.getClass())); | ||
| bytes = add(bytes, estimateStringOnce(uri.toString(), visited)); | ||
| bytes = add(bytes, estimateStringOnce(uri.getScheme(), visited)); | ||
| bytes = add(bytes, estimateStringOnce(uri.getUserInfo(), visited)); | ||
| bytes = add(bytes, estimateStringOnce(uri.getHost(), visited)); | ||
| bytes = add(bytes, estimateStringOnce(uri.getPath(), visited)); | ||
| bytes = add(bytes, estimateStringOnce(uri.getQuery(), visited)); | ||
| bytes = add(bytes, estimateStringOnce(uri.getFragment(), visited)); | ||
| bytes = add(bytes, estimateStringOnce(uri.getAuthority(), visited)); | ||
| bytes = add(bytes, estimateStringOnce(uri.getSchemeSpecificPart(), visited)); | ||
| } | ||
| return bytes; | ||
| } | ||
|
|
||
| private static long estimateStringOnce(String value, Set<Object> visited) { | ||
| return value == null || !visited.add(value) ? 0L : JvmSizeUtils.stringSize(value); | ||
| } | ||
|
|
||
| private static String location(Table table) { | ||
| if (table instanceof FormatTable) { | ||
| return ((FormatTable) table).location(); | ||
| } | ||
| if (table instanceof ObjectTable) { | ||
| return ((ObjectTable) table).location(); | ||
| } | ||
| if (table instanceof LanceTable) { | ||
| return ((LanceTable) table).location(); | ||
| } | ||
| if (table instanceof IcebergTable) { | ||
| return ((IcebergTable) table).location(); | ||
| } | ||
| return null; | ||
| } | ||
|
|
||
| private static long add(long left, long right) { | ||
| return JvmSizeUtils.saturatedAdd(left, right); | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -50,7 +50,6 @@ | |
| import org.apache.hadoop.hive.conf.HiveConf; | ||
| import org.apache.logging.log4j.LogManager; | ||
| import org.apache.logging.log4j.Logger; | ||
| import org.apache.paimon.catalog.CachingCatalog; | ||
| import org.apache.paimon.catalog.Catalog; | ||
| import org.apache.paimon.catalog.CatalogContext; | ||
| import org.apache.paimon.catalog.CatalogFactory; | ||
|
|
@@ -60,7 +59,6 @@ | |
| import org.apache.paimon.hive.HiveCatalogOptions; | ||
| import org.apache.paimon.options.CatalogOptions; | ||
| import org.apache.paimon.options.Options; | ||
| import org.apache.paimon.privilege.PrivilegedCatalog; | ||
|
|
||
| import java.io.IOException; | ||
| import java.io.UncheckedIOException; | ||
|
|
@@ -499,7 +497,7 @@ static boolean hasDlfCompatibleStorage(List<StorageProperties> storageProperties | |
| } | ||
|
|
||
| Options buildCatalogOptions() { | ||
| return PaimonCatalogFactory.buildCatalogOptions(catalogProps, metaCache.hasEnclosingWeightLimit()); | ||
| return PaimonCatalogFactory.buildCatalogOptions(catalogProps); | ||
| } | ||
|
|
||
| /** | ||
|
|
@@ -543,11 +541,15 @@ private Catalog createCatalogFromContext(CatalogContext catalogContext, String f | |
| try { | ||
| Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); | ||
| return context.executeAuthenticated(() -> { | ||
| // PaimonMetaCacheCatalog installs PrivilegedCatalog after the raw metadata cache. | ||
| Catalog catalog = PaimonCatalogProperties.HMS.equals(flavor) | ||
| ? createHmsCatalog(catalogContext, hmsAuth, catalogProps.getRaw(), | ||
| storageHadoopConfig) | ||
| : CatalogFactory.createCatalog(catalogContext); | ||
| return catalog; | ||
| : CatalogFactory.createUnwrappedCatalog(catalogContext, getClass().getClassLoader()); | ||
| return PaimonMetaCacheCatalog.tryToCreate(catalog, metaCache, | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [P2] Fence this cache-backed catalog publication against close A statement can retain this connector while |
||
| DEFAULT_TABLE_CACHE_CAPACITY, resolveTableCacheTtlSecond(catalogProps.getRaw()), | ||
| catalogContext.options(), PaimonCatalogFactory.isCatalogCacheEnabled(catalogProps), | ||
| metaCache.hasEnclosingWeightLimit()); | ||
| }); | ||
| } catch (Exception e) { | ||
| throw new RuntimeException(failureMessage + " (flavor=" + flavor + "): " + e.getMessage(), e); | ||
|
|
@@ -579,8 +581,7 @@ static Catalog createHmsCatalog(CatalogContext catalogContext, HadoopAuthenticat | |
| fileIO, hiveConf, clientClass, options, warehousePath.toUri().toString())); | ||
| catalog = PaimonHmsClientPool.install(catalog, hmsAuth); | ||
| catalog = PaimonHmsCatalog.install(catalog, properties, storageHadoopConfig); | ||
| catalog = CachingCatalog.tryToCreate(catalog, options); | ||
| return PrivilegedCatalog.tryToCreate(catalog, options); | ||
| return catalog; | ||
| } catch (IOException e) { | ||
| throw new UncheckedIOException(e); | ||
| } | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.