Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,8 @@
import org.apache.doris.common.jni.vec.TableSchema;
import org.apache.doris.common.security.authentication.PreExecutionAuthenticator;
import org.apache.doris.common.security.authentication.PreExecutionAuthenticatorCache;
import org.apache.doris.paimon.PaimonTableCache.PaimonTableCacheKey;
import org.apache.doris.paimon.PaimonTableCache.TableExt;

import com.google.common.base.Preconditions;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.reader.RecordReader;
Expand All @@ -51,29 +50,14 @@ public class PaimonJniScanner extends JniScanner {

private final Map<String, String> params;
@Deprecated
private final Map<String, String> paimonOptionParams;
@Deprecated
private final Map<String, String> hadoopOptionParams;
@Deprecated
private final String dbName;
@Deprecated
private final String tblName;
private final String paimonSplit;
private final String paimonPredicate;
private Table table;
private RecordReader<InternalRow> reader;
private final PaimonColumnValue columnValue = new PaimonColumnValue();
private List<String> paimonAllFieldNames;
private List<DataType> paimonDataTypeList;

@Deprecated
private long ctlId;
@Deprecated
private long dbId;
@Deprecated
private long tblId;
@Deprecated
private long lastUpdateTime;
private RecordReader.RecordIterator<InternalRow> recordIterator = null;
private final ClassLoader classLoader;
private PreExecutionAuthenticator preExecutionAuthenticator;
Expand All @@ -92,17 +76,7 @@ public PaimonJniScanner(int batchSize, Map<String, String> params) {
}
paimonSplit = params.get("paimon_split");
paimonPredicate = params.get("paimon_predicate");
dbName = params.get("db_name");
tblName = params.get("table_name");
ctlId = Long.parseLong(params.get("ctl_id"));
dbId = Long.parseLong(params.get("db_id"));
tblId = Long.parseLong(params.get("tbl_id"));
lastUpdateTime = Long.parseLong(params.get("last_update_time"));
initTableInfo(columnTypes, requiredFields, batchSize);
paimonOptionParams = params.entrySet().stream()
.filter(kv -> kv.getKey().startsWith(PAIMON_OPTION_PREFIX))
.collect(Collectors
.toMap(kv1 -> kv1.getKey().substring(PAIMON_OPTION_PREFIX.length()), kv1 -> kv1.getValue()));
hadoopOptionParams = params.entrySet().stream()
.filter(kv -> kv.getKey().startsWith(HADOOP_OPTION_PREFIX))
.collect(Collectors
Expand Down Expand Up @@ -242,20 +216,8 @@ protected TableSchema parseTableSchema() throws UnsupportedOperationException {
}

private void initTable() {
if (params.containsKey("serialized_table")) {
table = PaimonUtils.deserialize(params.get("serialized_table"));
} else {
PaimonTableCacheKey key = new PaimonTableCacheKey(ctlId, dbId, tblId,
paimonOptionParams, hadoopOptionParams, dbName, tblName);
TableExt tableExt = PaimonTableCache.getTable(key);
if (tableExt.getCreateTime() < lastUpdateTime) {
LOG.warn("invalidate cache table:{}, localTime:{}, remoteTime:{}", key, tableExt.getCreateTime(),
lastUpdateTime);
PaimonTableCache.invalidateTableCache(key);
tableExt = PaimonTableCache.getTable(key);
}
this.table = tableExt.getTable();
}
Preconditions.checkState(params.containsKey("serialized_table"));
table = PaimonUtils.deserialize(params.get("serialized_table"));
paimonAllFieldNames = PaimonUtils.getFieldNames(this.table.rowType());
if (LOG.isDebugEnabled()) {
LOG.debug("paimonAllFieldNames:{}", paimonAllFieldNames);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ private void parseVirtualHostedStyleUri() throws UserException {
if (!StringUtils.isEmpty(path) && !"/".equals(path)) {
key = path.substring(1);
} else {
throw new UserException("missing key: " + this.uri);
throw new UserException("missing key from uri: " + this.uri);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -186,9 +186,10 @@ public Map<String, String> getBackendStorageProperties() {

for (StorageProperties sp : storageMap.values()) {
Map<String, String> backendProps = sp.getBackendConfigProperties();
if (backendProps != null) {
result.putAll(backendProps);
}
// the backend property's value can not be null, because it will be serialized to thrift,
// which does not support null value.
backendProps.entrySet().stream().filter(e -> e.getValue() != null)
.forEach(e -> result.put(e.getKey(), e.getValue()));
}

this.backendStorageProperties = result;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.datasource.credentials;

import java.util.Map;

/**
* Interface for future FileIO-based credential extraction.
* This interface is designed to be compatible with Iceberg/Paimon FileIO
*/
public interface CredentialExtractor {
// AWS credential property keys for backend
String BACKEND_AWS_ACCESS_KEY = "AWS_ACCESS_KEY";
String BACKEND_AWS_SECRET_KEY = "AWS_SECRET_KEY";
String BACKEND_AWS_TOKEN = "AWS_TOKEN";
String BACKEND_AWS_ENDPOINT = "AWS_ENDPOINT";
String BACKEND_AWS_REGION = "AWS_REGION";

/**
* Extract credentials from a generic properties map.
*
* @param properties properties map from any source (FileIO, Table IO, etc.)
* @return extracted credentials as backend properties
*/
Map<String, String> extractCredentials(Map<String, String> properties);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.datasource.credentials;

import com.google.common.annotations.VisibleForTesting;

import java.util.Map;

public class CredentialUtils {
/**
* Future method for FileIO-based credential extraction.
* This method signature is designed to be compatible with future FileIO implementations.
*
* @param fileIoProperties properties from FileIO (reserved for future use)
* @param extractor custom credential extractor
* @return extracted credentials
*/
@VisibleForTesting
public static Map<String, String> extractCredentialsFromFileIO(Map<String, String> fileIoProperties,
CredentialExtractor extractor) {
return extractor.extractCredentials(fileIoProperties);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.datasource.iceberg;

import org.apache.doris.datasource.credentials.CredentialExtractor;

import com.google.common.collect.Maps;
import org.apache.iceberg.aws.s3.S3FileIOProperties;

import java.util.Map;

/**
* Default credential extractor for S3 credentials.
*/
public class IcebergS3CredentialExtractor implements CredentialExtractor {

@Override
public Map<String, String> extractCredentials(Map<String, String> properties) {
Map<String, String> credentials = Maps.newHashMap();

if (properties == null || properties.isEmpty()) {
return credentials;
}

// Extract AWS credentials from Iceberg S3 FileIO format
if (properties.containsKey(S3FileIOProperties.ACCESS_KEY_ID)) {
credentials.put(BACKEND_AWS_ACCESS_KEY, properties.get(S3FileIOProperties.ACCESS_KEY_ID));
}
if (properties.containsKey(S3FileIOProperties.SECRET_ACCESS_KEY)) {
credentials.put(BACKEND_AWS_SECRET_KEY, properties.get(S3FileIOProperties.SECRET_ACCESS_KEY));
}
if (properties.containsKey(S3FileIOProperties.SESSION_TOKEN)) {
credentials.put(BACKEND_AWS_TOKEN, properties.get(S3FileIOProperties.SESSION_TOKEN));
}

return credentials;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,63 +22,17 @@
import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.doris.datasource.property.storage.StorageProperties.Type;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Maps;
import org.apache.iceberg.Table;
import org.apache.iceberg.aws.s3.S3FileIOProperties;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.util.Map;

public class IcebergVendedCredentialsProvider {
private static final Logger LOG = LogManager.getLogger(IcebergVendedCredentialsProvider.class);

// AWS credential property keys for backend
private static final String BACKEND_AWS_ACCESS_KEY = "AWS_ACCESS_KEY";
private static final String BACKEND_AWS_SECRET_KEY = "AWS_SECRET_KEY";
private static final String BACKEND_AWS_TOKEN = "AWS_TOKEN";

/**
* Interface for future FileIO-based credential extraction.
* This interface is designed to be compatible with Iceberg FileIO
* when we implement FileIO support in the future.
*/
public interface CredentialExtractor {
/**
* Extract credentials from a generic properties map.
*
* @param properties properties map from any source (FileIO, Table IO, etc.)
* @return extracted credentials as backend properties
*/
Map<String, String> extractCredentials(Map<String, String> properties);
}

/**
* Default credential extractor for S3 credentials.
*/
public static class S3CredentialExtractor implements CredentialExtractor {
@Override
public Map<String, String> extractCredentials(Map<String, String> properties) {
Map<String, String> credentials = Maps.newHashMap();

if (properties == null || properties.isEmpty()) {
return credentials;
}

// Extract AWS credentials from Iceberg S3 FileIO format
if (properties.containsKey(S3FileIOProperties.ACCESS_KEY_ID)) {
credentials.put(BACKEND_AWS_ACCESS_KEY, properties.get(S3FileIOProperties.ACCESS_KEY_ID));
}
if (properties.containsKey(S3FileIOProperties.SECRET_ACCESS_KEY)) {
credentials.put(BACKEND_AWS_SECRET_KEY, properties.get(S3FileIOProperties.SECRET_ACCESS_KEY));
}
if (properties.containsKey(S3FileIOProperties.SESSION_TOKEN)) {
credentials.put(BACKEND_AWS_TOKEN, properties.get(S3FileIOProperties.SESSION_TOKEN));
}

return credentials;
}
}

private static final S3CredentialExtractor s3Extractor = new S3CredentialExtractor();
private static final IcebergS3CredentialExtractor s3Extractor = new IcebergS3CredentialExtractor();

/**
* Extract vended credentials from Iceberg Table and convert to backend properties.
Expand All @@ -95,20 +49,6 @@ public static Map<String, String> extractVendedCredentialsFromTable(Table table)
return s3Extractor.extractCredentials(ioProperties);
}

/**
* Future method for FileIO-based credential extraction.
* This method signature is designed to be compatible with future FileIO implementations.
*
* @param fileIoProperties properties from FileIO (reserved for future use)
* @param extractor custom credential extractor
* @return extracted credentials
*/
@VisibleForTesting
public static Map<String, String> extractCredentialsFromFileIO(Map<String, String> fileIoProperties,
CredentialExtractor extractor) {
return extractor.extractCredentials(fileIoProperties);
}

/**
* Get backend location properties for Iceberg catalog with optional vended credentials support.
* This method extracts the duplicate logic from IcebergScanNode and IcebergTableSink.
Expand All @@ -135,6 +75,10 @@ public static Map<String, String> getBackendLocationProperties(
locationProperties.putAll(storageProperties.getBackendConfigProperties());
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("Iceberg backend location properties: {}, vendedCredentialsEnabled: {}, icebergTable: {}",
locationProperties, vendedCredentialsEnabled, icebergTable != null ? icebergTable.name() : "null");
}
return locationProperties;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import java.util.Map;

@Deprecated
public class PaimonDLFExternalCatalog extends PaimonExternalCatalog {

public PaimonDLFExternalCatalog(long catalogId, String name, String resource,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,14 @@
import java.util.List;
import java.util.Map;

// The subclasses of this class are all deprecated, only for meta persistence compatibility.
public class PaimonExternalCatalog extends ExternalCatalog {
private static final Logger LOG = LogManager.getLogger(PaimonExternalCatalog.class);
public static final String PAIMON_CATALOG_TYPE = "paimon.catalog.type";
public static final String PAIMON_FILESYSTEM = "filesystem";
public static final String PAIMON_HMS = "hms";
public static final String PAIMON_DLF = "dlf";
public static final String PAIMON_REST = "rest";
protected String catalogType;
protected Catalog catalog;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ public static ExternalCatalog createCatalog(long catalogId, String name, String
case PaimonExternalCatalog.PAIMON_HMS:
case PaimonExternalCatalog.PAIMON_FILESYSTEM:
case PaimonExternalCatalog.PAIMON_DLF:
case PaimonExternalCatalog.PAIMON_REST:
return new PaimonExternalCatalog(catalogId, name, resource, props, comment);
default:
throw new DdlException("Unknown " + PaimonExternalCatalog.PAIMON_CATALOG_TYPE
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,14 +108,16 @@ public TTableDescriptor toThrift() {
List<Column> schema = getFullSchema();
if (PaimonExternalCatalog.PAIMON_HMS.equals(getPaimonCatalogType())
|| PaimonExternalCatalog.PAIMON_FILESYSTEM.equals(getPaimonCatalogType())
|| PaimonExternalCatalog.PAIMON_DLF.equals(getPaimonCatalogType())) {
|| PaimonExternalCatalog.PAIMON_DLF.equals(getPaimonCatalogType())
|| PaimonExternalCatalog.PAIMON_REST.equals(getPaimonCatalogType())) {
THiveTable tHiveTable = new THiveTable(dbName, name, new HashMap<>());
TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(), TTableType.HIVE_TABLE, schema.size(), 0,
getName(), dbName);
tTableDescriptor.setHiveTable(tHiveTable);
return tTableDescriptor;
} else {
throw new IllegalArgumentException("Currently only supports hms/filesystem catalog,not support :"
throw new IllegalArgumentException(
"Currently only supports hms/dlf/rest/filesystem catalog, do not support :"
+ getPaimonCatalogType());
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import java.util.Map;

@Deprecated
public class PaimonFileExternalCatalog extends PaimonExternalCatalog {

public PaimonFileExternalCatalog(long catalogId, String name, String resource,
Expand Down
Loading