diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java index 6ba4578dc6ec10..d25d5f7b94f4aa 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java @@ -22,9 +22,8 @@ import org.apache.doris.common.jni.vec.TableSchema; import org.apache.doris.common.security.authentication.PreExecutionAuthenticator; import org.apache.doris.common.security.authentication.PreExecutionAuthenticatorCache; -import org.apache.doris.paimon.PaimonTableCache.PaimonTableCacheKey; -import org.apache.doris.paimon.PaimonTableCache.TableExt; +import com.google.common.base.Preconditions; import org.apache.paimon.data.InternalRow; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.reader.RecordReader; @@ -51,13 +50,7 @@ public class PaimonJniScanner extends JniScanner { private final Map params; @Deprecated - private final Map paimonOptionParams; - @Deprecated private final Map hadoopOptionParams; - @Deprecated - private final String dbName; - @Deprecated - private final String tblName; private final String paimonSplit; private final String paimonPredicate; private Table table; @@ -65,15 +58,6 @@ public class PaimonJniScanner extends JniScanner { private final PaimonColumnValue columnValue = new PaimonColumnValue(); private List paimonAllFieldNames; private List paimonDataTypeList; - - @Deprecated - private long ctlId; - @Deprecated - private long dbId; - @Deprecated - private long tblId; - @Deprecated - private long lastUpdateTime; private RecordReader.RecordIterator recordIterator = null; private final ClassLoader classLoader; private PreExecutionAuthenticator preExecutionAuthenticator; @@ -92,17 +76,7 @@ public PaimonJniScanner(int batchSize, Map params) { } paimonSplit = params.get("paimon_split"); paimonPredicate = params.get("paimon_predicate"); - dbName = params.get("db_name"); - tblName = params.get("table_name"); - ctlId = Long.parseLong(params.get("ctl_id")); - dbId = Long.parseLong(params.get("db_id")); - tblId = Long.parseLong(params.get("tbl_id")); - lastUpdateTime = Long.parseLong(params.get("last_update_time")); initTableInfo(columnTypes, requiredFields, batchSize); - paimonOptionParams = params.entrySet().stream() - .filter(kv -> kv.getKey().startsWith(PAIMON_OPTION_PREFIX)) - .collect(Collectors - .toMap(kv1 -> kv1.getKey().substring(PAIMON_OPTION_PREFIX.length()), kv1 -> kv1.getValue())); hadoopOptionParams = params.entrySet().stream() .filter(kv -> kv.getKey().startsWith(HADOOP_OPTION_PREFIX)) .collect(Collectors @@ -242,20 +216,8 @@ protected TableSchema parseTableSchema() throws UnsupportedOperationException { } private void initTable() { - if (params.containsKey("serialized_table")) { - table = PaimonUtils.deserialize(params.get("serialized_table")); - } else { - PaimonTableCacheKey key = new PaimonTableCacheKey(ctlId, dbId, tblId, - paimonOptionParams, hadoopOptionParams, dbName, tblName); - TableExt tableExt = PaimonTableCache.getTable(key); - if (tableExt.getCreateTime() < lastUpdateTime) { - LOG.warn("invalidate cache table:{}, localTime:{}, remoteTime:{}", key, tableExt.getCreateTime(), - lastUpdateTime); - PaimonTableCache.invalidateTableCache(key); - tableExt = PaimonTableCache.getTable(key); - } - this.table = tableExt.getTable(); - } + Preconditions.checkState(params.containsKey("serialized_table")); + table = PaimonUtils.deserialize(params.get("serialized_table")); paimonAllFieldNames = PaimonUtils.getFieldNames(this.table.rowType()); if (LOG.isDebugEnabled()) { LOG.debug("paimonAllFieldNames:{}", paimonAllFieldNames); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java index 5d1d63b8c7fa6b..5ef6e4024df5f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java @@ -254,7 +254,7 @@ private void parseVirtualHostedStyleUri() throws UserException { if (!StringUtils.isEmpty(path) && !"/".equals(path)) { key = path.substring(1); } else { - throw new UserException("missing key: " + this.uri); + throw new UserException("missing key from uri: " + this.uri); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java index da598e982e6067..502d558b69834b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java @@ -170,9 +170,10 @@ public Map getBackendStorageProperties() { for (StorageProperties sp : storageMap.values()) { Map backendProps = sp.getBackendConfigProperties(); - if (backendProps != null) { - result.putAll(backendProps); - } + // the backend property's value can not be null, because it will be serialized to thrift, + // which does not support null value. + backendProps.entrySet().stream().filter(e -> e.getValue() != null) + .forEach(e -> result.put(e.getKey(), e.getValue())); } this.backendStorageProperties = result; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/CredentialExtractor.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/CredentialExtractor.java new file mode 100644 index 00000000000000..049bad3d3b7a08 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/CredentialExtractor.java @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.credentials; + +import java.util.Map; + +/** + * Interface for future FileIO-based credential extraction. + * This interface is designed to be compatible with Iceberg/Paimon FileIO + */ +public interface CredentialExtractor { + // AWS credential property keys for backend + String BACKEND_AWS_ACCESS_KEY = "AWS_ACCESS_KEY"; + String BACKEND_AWS_SECRET_KEY = "AWS_SECRET_KEY"; + String BACKEND_AWS_TOKEN = "AWS_TOKEN"; + String BACKEND_AWS_ENDPOINT = "AWS_ENDPOINT"; + String BACKEND_AWS_REGION = "AWS_REGION"; + + /** + * Extract credentials from a generic properties map. + * + * @param properties properties map from any source (FileIO, Table IO, etc.) + * @return extracted credentials as backend properties + */ + Map extractCredentials(Map properties); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/CredentialUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/CredentialUtils.java new file mode 100644 index 00000000000000..1f7060a8c627bf --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/CredentialUtils.java @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.credentials; + +import com.google.common.annotations.VisibleForTesting; + +import java.util.Map; + +public class CredentialUtils { + /** + * Future method for FileIO-based credential extraction. + * This method signature is designed to be compatible with future FileIO implementations. + * + * @param fileIoProperties properties from FileIO (reserved for future use) + * @param extractor custom credential extractor + * @return extracted credentials + */ + @VisibleForTesting + public static Map extractCredentialsFromFileIO(Map fileIoProperties, + CredentialExtractor extractor) { + return extractor.extractCredentials(fileIoProperties); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergS3CredentialExtractor.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergS3CredentialExtractor.java new file mode 100644 index 00000000000000..d3ad00928feac3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergS3CredentialExtractor.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.doris.datasource.credentials.CredentialExtractor; + +import com.google.common.collect.Maps; +import org.apache.iceberg.aws.s3.S3FileIOProperties; + +import java.util.Map; + +/** + * Default credential extractor for S3 credentials. + */ +public class IcebergS3CredentialExtractor implements CredentialExtractor { + + @Override + public Map extractCredentials(Map properties) { + Map credentials = Maps.newHashMap(); + + if (properties == null || properties.isEmpty()) { + return credentials; + } + + // Extract AWS credentials from Iceberg S3 FileIO format + if (properties.containsKey(S3FileIOProperties.ACCESS_KEY_ID)) { + credentials.put(BACKEND_AWS_ACCESS_KEY, properties.get(S3FileIOProperties.ACCESS_KEY_ID)); + } + if (properties.containsKey(S3FileIOProperties.SECRET_ACCESS_KEY)) { + credentials.put(BACKEND_AWS_SECRET_KEY, properties.get(S3FileIOProperties.SECRET_ACCESS_KEY)); + } + if (properties.containsKey(S3FileIOProperties.SESSION_TOKEN)) { + credentials.put(BACKEND_AWS_TOKEN, properties.get(S3FileIOProperties.SESSION_TOKEN)); + } + + return credentials; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergVendedCredentialsProvider.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergVendedCredentialsProvider.java index 06a8b40e2b7907..72c137104f33db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergVendedCredentialsProvider.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergVendedCredentialsProvider.java @@ -22,63 +22,17 @@ import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.datasource.property.storage.StorageProperties.Type; -import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Maps; import org.apache.iceberg.Table; -import org.apache.iceberg.aws.s3.S3FileIOProperties; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.Map; public class IcebergVendedCredentialsProvider { + private static final Logger LOG = LogManager.getLogger(IcebergVendedCredentialsProvider.class); - // AWS credential property keys for backend - private static final String BACKEND_AWS_ACCESS_KEY = "AWS_ACCESS_KEY"; - private static final String BACKEND_AWS_SECRET_KEY = "AWS_SECRET_KEY"; - private static final String BACKEND_AWS_TOKEN = "AWS_TOKEN"; - - /** - * Interface for future FileIO-based credential extraction. - * This interface is designed to be compatible with Iceberg FileIO - * when we implement FileIO support in the future. - */ - public interface CredentialExtractor { - /** - * Extract credentials from a generic properties map. - * - * @param properties properties map from any source (FileIO, Table IO, etc.) - * @return extracted credentials as backend properties - */ - Map extractCredentials(Map properties); - } - - /** - * Default credential extractor for S3 credentials. - */ - public static class S3CredentialExtractor implements CredentialExtractor { - @Override - public Map extractCredentials(Map properties) { - Map credentials = Maps.newHashMap(); - - if (properties == null || properties.isEmpty()) { - return credentials; - } - - // Extract AWS credentials from Iceberg S3 FileIO format - if (properties.containsKey(S3FileIOProperties.ACCESS_KEY_ID)) { - credentials.put(BACKEND_AWS_ACCESS_KEY, properties.get(S3FileIOProperties.ACCESS_KEY_ID)); - } - if (properties.containsKey(S3FileIOProperties.SECRET_ACCESS_KEY)) { - credentials.put(BACKEND_AWS_SECRET_KEY, properties.get(S3FileIOProperties.SECRET_ACCESS_KEY)); - } - if (properties.containsKey(S3FileIOProperties.SESSION_TOKEN)) { - credentials.put(BACKEND_AWS_TOKEN, properties.get(S3FileIOProperties.SESSION_TOKEN)); - } - - return credentials; - } - } - - private static final S3CredentialExtractor s3Extractor = new S3CredentialExtractor(); + private static final IcebergS3CredentialExtractor s3Extractor = new IcebergS3CredentialExtractor(); /** * Extract vended credentials from Iceberg Table and convert to backend properties. @@ -95,20 +49,6 @@ public static Map extractVendedCredentialsFromTable(Table table) return s3Extractor.extractCredentials(ioProperties); } - /** - * Future method for FileIO-based credential extraction. - * This method signature is designed to be compatible with future FileIO implementations. - * - * @param fileIoProperties properties from FileIO (reserved for future use) - * @param extractor custom credential extractor - * @return extracted credentials - */ - @VisibleForTesting - public static Map extractCredentialsFromFileIO(Map fileIoProperties, - CredentialExtractor extractor) { - return extractor.extractCredentials(fileIoProperties); - } - /** * Get backend location properties for Iceberg catalog with optional vended credentials support. * This method extracts the duplicate logic from IcebergScanNode and IcebergTableSink. @@ -135,6 +75,10 @@ public static Map getBackendLocationProperties( locationProperties.putAll(storageProperties.getBackendConfigProperties()); } } + if (LOG.isDebugEnabled()) { + LOG.debug("Iceberg backend location properties: {}, vendedCredentialsEnabled: {}, icebergTable: {}", + locationProperties, vendedCredentialsEnabled, icebergTable != null ? icebergTable.name() : "null"); + } return locationProperties; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonDLFExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonDLFExternalCatalog.java index fba59545922288..a982abe5b017f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonDLFExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonDLFExternalCatalog.java @@ -19,6 +19,7 @@ import java.util.Map; +@Deprecated public class PaimonDLFExternalCatalog extends PaimonExternalCatalog { public PaimonDLFExternalCatalog(long catalogId, String name, String resource, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java index b8decdc032d408..3e58e654cdae67 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java @@ -39,12 +39,14 @@ import java.util.List; import java.util.Map; +// The subclasses of this class are all deprecated, only for meta persistence compatibility. public class PaimonExternalCatalog extends ExternalCatalog { private static final Logger LOG = LogManager.getLogger(PaimonExternalCatalog.class); public static final String PAIMON_CATALOG_TYPE = "paimon.catalog.type"; public static final String PAIMON_FILESYSTEM = "filesystem"; public static final String PAIMON_HMS = "hms"; public static final String PAIMON_DLF = "dlf"; + public static final String PAIMON_REST = "rest"; protected String catalogType; protected Catalog catalog; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogFactory.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogFactory.java index b904ac169ff3b3..ae19907d93219e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogFactory.java @@ -37,6 +37,7 @@ public static ExternalCatalog createCatalog(long catalogId, String name, String case PaimonExternalCatalog.PAIMON_HMS: case PaimonExternalCatalog.PAIMON_FILESYSTEM: case PaimonExternalCatalog.PAIMON_DLF: + case PaimonExternalCatalog.PAIMON_REST: return new PaimonExternalCatalog(catalogId, name, resource, props, comment); default: throw new DdlException("Unknown " + PaimonExternalCatalog.PAIMON_CATALOG_TYPE diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index 371ce3864dd1dc..1b4a9700aca70f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -108,14 +108,16 @@ public TTableDescriptor toThrift() { List schema = getFullSchema(); if (PaimonExternalCatalog.PAIMON_HMS.equals(getPaimonCatalogType()) || PaimonExternalCatalog.PAIMON_FILESYSTEM.equals(getPaimonCatalogType()) - || PaimonExternalCatalog.PAIMON_DLF.equals(getPaimonCatalogType())) { + || PaimonExternalCatalog.PAIMON_DLF.equals(getPaimonCatalogType()) + || PaimonExternalCatalog.PAIMON_REST.equals(getPaimonCatalogType())) { THiveTable tHiveTable = new THiveTable(dbName, name, new HashMap<>()); TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(), TTableType.HIVE_TABLE, schema.size(), 0, getName(), dbName); tTableDescriptor.setHiveTable(tHiveTable); return tTableDescriptor; } else { - throw new IllegalArgumentException("Currently only supports hms/filesystem catalog,not support :" + throw new IllegalArgumentException( + "Currently only supports hms/dlf/rest/filesystem catalog, do not support :" + getPaimonCatalogType()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java index c2c3e1395675e4..9bc233f4b05f15 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java @@ -19,6 +19,7 @@ import java.util.Map; +@Deprecated public class PaimonFileExternalCatalog extends PaimonExternalCatalog { public PaimonFileExternalCatalog(long catalogId, String name, String resource, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonHMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonHMSExternalCatalog.java index ae5cf7073ddbab..0a4702ab4f1cd1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonHMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonHMSExternalCatalog.java @@ -19,6 +19,7 @@ import java.util.Map; +@Deprecated public class PaimonHMSExternalCatalog extends PaimonExternalCatalog { public PaimonHMSExternalCatalog(long catalogId, String name, String resource, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonOssCredentialExtractor.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonOssCredentialExtractor.java new file mode 100644 index 00000000000000..b5068e7780aa80 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonOssCredentialExtractor.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.datasource.credentials.CredentialExtractor; +import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties; +import org.apache.doris.datasource.property.storage.OSSProperties; + +import com.google.common.collect.Maps; + +import java.util.Map; +import java.util.Optional; + +/** + * Default credential extractor for S3 credentials. + */ +public class PaimonOssCredentialExtractor implements CredentialExtractor { + // oss credential property keys from DLF Paimon FileIO + private static final String OSS_ACCESS_KEY = "fs.oss.accessKeyId"; + private static final String OSS_SECRET_KEY = "fs.oss.accessKeySecret"; + private static final String OSS_SECURITY_TOKEN = "fs.oss.securityToken"; + private static final String OSS_ENDPOINT = "fs.oss.endpoint"; + + @Override + public Map extractCredentials(Map properties) { + Map credentials = Maps.newHashMap(); + + if (properties == null || properties.isEmpty()) { + return credentials; + } + + // Extract AWS credentials from Iceberg S3 FileIO format + if (properties.containsKey(OSS_ACCESS_KEY)) { + credentials.put(BACKEND_AWS_ACCESS_KEY, properties.get(OSS_ACCESS_KEY)); + } + if (properties.containsKey(OSS_SECRET_KEY)) { + credentials.put(BACKEND_AWS_SECRET_KEY, properties.get(OSS_SECRET_KEY)); + } + if (properties.containsKey(OSS_SECURITY_TOKEN)) { + credentials.put(BACKEND_AWS_TOKEN, properties.get(OSS_SECURITY_TOKEN)); + } + if (properties.containsKey(OSS_ENDPOINT)) { + credentials.put(BACKEND_AWS_ENDPOINT, properties.get(OSS_ENDPOINT)); + Optional regionOpt = AbstractS3CompatibleProperties.extractRegion( + OSSProperties.ENDPOINT_PATTERN, properties.get(OSS_ENDPOINT)); + if (regionOpt.isPresent()) { + credentials.put(BACKEND_AWS_REGION, regionOpt.get()); + } + } + + return credentials; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonVendedCredentialsProvider.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonVendedCredentialsProvider.java new file mode 100644 index 00000000000000..375671fb1d2828 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonVendedCredentialsProvider.java @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.datasource.property.metastore.MetastoreProperties; +import org.apache.doris.datasource.property.metastore.PaimonRestMetaStoreProperties; +import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.datasource.property.storage.StorageProperties.Type; + +import com.google.common.collect.Maps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.paimon.rest.RESTToken; +import org.apache.paimon.rest.RESTTokenFileIO; +import org.apache.paimon.table.Table; + +import java.util.Map; + +public class PaimonVendedCredentialsProvider { + private static final Logger LOG = LogManager.getLogger(PaimonVendedCredentialsProvider.class); + + private static final PaimonOssCredentialExtractor ossExtractor = new PaimonOssCredentialExtractor(); + + /** + * Extract vended credentials from Paimon Table and convert to backend properties. + * + * @param table the Paimon table + * @return Map of backend properties with credentials + */ + public static Map extractVendedCredentialsFromTable(String tokenProvider, Table table) { + if (table == null || table.fileIO() == null) { + return Maps.newHashMap(); + } + + if (!(table.fileIO() instanceof RESTTokenFileIO)) { + if (LOG.isDebugEnabled()) { + LOG.debug("File IO of table {} is not RESTTokenFileIO, cannot extract vended credentials: {}", + table.name(), table.fileIO().getClass().getName()); + } + return Maps.newHashMap(); + } + + RESTTokenFileIO restTokenFileIO = (RESTTokenFileIO) table.fileIO(); + RESTToken restToken = restTokenFileIO.validToken(); + Map tokens = restToken.token(); + if ("dlf".equalsIgnoreCase(tokenProvider)) { + return ossExtractor.extractCredentials(tokens); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Unsupported token provider: {} for table {}, cannot extract vended credentials", + tokenProvider, table.name()); + } + return Maps.newHashMap(); + } + } + + /** + * Get backend location properties for Paimon catalog with optional vended credentials support. + * Provides backend location properties for a Paimon catalog, including support for vended credentials + * + * @param storagePropertiesMap Map of storage properties + * @param paimonTable Optional Paimon table for vended credentials extraction + * @return Map of backend location properties + */ + public static Map getBackendLocationProperties( + MetastoreProperties metastoreProperties, + Map storagePropertiesMap, + Table paimonTable) { + Map vendedCredentials = Maps.newHashMap(); + if (metastoreProperties instanceof PaimonRestMetaStoreProperties) { + String tokenProvider = ((PaimonRestMetaStoreProperties) metastoreProperties).getTokenProvider(); + vendedCredentials = extractVendedCredentialsFromTable(tokenProvider, paimonTable); + } + + Map locationProperties = Maps.newHashMap(); + for (StorageProperties storageProperties : storagePropertiesMap.values()) { + locationProperties.putAll(storageProperties.getBackendConfigProperties(vendedCredentials)); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Extracted backend location properties: {} for Paimon table: {}", + locationProperties, paimonTable != null ? paimonTable.name() : "null"); + } + return locationProperties; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index 807bccf7aa534e..55984aa42bac09 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -31,6 +31,7 @@ import org.apache.doris.datasource.paimon.PaimonExternalCatalog; import org.apache.doris.datasource.paimon.PaimonExternalTable; import org.apache.doris.datasource.paimon.PaimonUtil; +import org.apache.doris.datasource.paimon.PaimonVendedCredentialsProvider; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.qe.SessionVariable; import org.apache.doris.spi.Split; @@ -451,7 +452,12 @@ public TableIf getTargetTable() { @Override protected Map getLocationProperties() { - return source.getCatalog().getCatalogProperty().getBackendStorageProperties(); + PaimonExternalCatalog catalog = (PaimonExternalCatalog) source.getCatalog(); + return PaimonVendedCredentialsProvider.getBackendLocationProperties( + catalog.getCatalogProperty().getMetastoreProperties(), + catalog.getCatalogProperty().getStoragePropertiesMap(), + source.getPaimonTable() + ); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java index b5e12a4970ca64..7602758d2b067d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java @@ -85,7 +85,6 @@ protected void appendS3PropertiesIsNeeded(List storagePropert catalogOptions.set("s3.endpoint", s3Properties.getEndpoint()); catalogOptions.set("s3.region", s3Properties.getRegion()); } - } protected void appendCatalogOptions(List storagePropertiesList) { @@ -93,6 +92,8 @@ protected void appendCatalogOptions(List storagePropertiesLis catalogOptions.set(CatalogOptions.WAREHOUSE.key(), warehouse); } catalogOptions.set(CatalogOptions.METASTORE.key(), getMetastoreType()); + + // FIXME(cmy): Rethink these custom properties origProps.forEach((k, v) -> { if (k.toLowerCase().startsWith(USER_PROPERTY_PREFIX)) { String newKey = k.substring(USER_PROPERTY_PREFIX.length()); @@ -101,6 +102,7 @@ protected void appendCatalogOptions(List storagePropertiesLis } } }); + appendS3PropertiesIsNeeded(storagePropertiesList); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonAliyunDLFMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonAliyunDLFMetaStoreProperties.java index 2f65dbad9d820b..2516cd6e4d0f26 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonAliyunDLFMetaStoreProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonAliyunDLFMetaStoreProperties.java @@ -105,7 +105,6 @@ public Catalog initializeCatalog(String catalogName, List sto return CatalogFactory.createCatalog(catalogContext); } - @Override protected void appendCustomCatalogOptions() { catalogOptions.set("metastore.client.class", ProxyMetaStoreClient.class.getName()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonPropertiesFactory.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonPropertiesFactory.java index 12dac3bb739189..958daafd175be4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonPropertiesFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonPropertiesFactory.java @@ -28,6 +28,7 @@ public PaimonPropertiesFactory() { register("dlf", PaimonAliyunDLFMetaStoreProperties::new); register("filesystem", PaimonFileSystemMetaStoreProperties::new); register("hms", PaimonHMSMetaStoreProperties::new); + register("rest", PaimonRestMetaStoreProperties::new); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonRestMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonRestMetaStoreProperties.java new file mode 100644 index 00000000000000..91e47c102530e5 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonRestMetaStoreProperties.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.metastore; + +import org.apache.doris.datasource.paimon.PaimonExternalCatalog; +import org.apache.doris.datasource.property.ConnectorProperty; +import org.apache.doris.datasource.property.ParamRules; +import org.apache.doris.datasource.property.storage.StorageProperties; + +import lombok.Getter; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; + +import java.util.List; +import java.util.Map; + +public class PaimonRestMetaStoreProperties extends AbstractPaimonProperties { + + private static final String PAIMON_REST_PROPERTY_PREFIX = "paimon.rest."; + + @ConnectorProperty(names = {"paimon.rest.uri", "uri"}, + description = "The uri of the Paimon rest catalog service.") + private String paimonRestUri = ""; + + @Getter + @ConnectorProperty( + names = {"paimon.rest.token.provider"}, + description = "the token provider for Paimon REST metastore, e.g., 'dlf' for Aliyun DLF." + ) + protected String tokenProvider = ""; + + // The following properties are specific to DLF rest catalog + @ConnectorProperty( + names = {"paimon.rest.dlf.access-key-id"}, + required = false, + description = "The access key ID for DLF, required when using DLF as token provider." + ) + protected String paimonRestDlfAccessKey = ""; + + @ConnectorProperty( + names = {"paimon.rest.dlf.access-key-secret"}, + required = false, + description = "The secret key secret for DLF, required when using DLF as token provider." + ) + protected String paimonRestDlfSecretKey = ""; + + protected PaimonRestMetaStoreProperties(Map props) { + super(props); + } + + @Override + public void initNormalizeAndCheckProps() { + super.initNormalizeAndCheckProps(); + buildRules().validate(); + } + + @Override + public String getPaimonCatalogType() { + return PaimonExternalCatalog.PAIMON_REST; + } + + @Override + public Catalog initializeCatalog(String catalogName, List storagePropertiesList) { + buildCatalogOptions(storagePropertiesList); + CatalogContext catalogContext = CatalogContext.create(catalogOptions); + return CatalogFactory.createCatalog(catalogContext); + } + + @Override + protected void appendCustomCatalogOptions() { + catalogOptions.set("uri", paimonRestUri); + for (Map.Entry entry : origProps.entrySet()) { + if (entry.getKey().startsWith(PAIMON_REST_PROPERTY_PREFIX)) { + String key = entry.getKey().substring(PAIMON_REST_PROPERTY_PREFIX.length()); + catalogOptions.set(key, entry.getValue()); + } + } + } + + @Override + protected String getMetastoreType() { + return "rest"; + } + + private ParamRules buildRules() { + ParamRules rules = new ParamRules(); + // Check for dlf rest catalog + rules.requireIf(tokenProvider, "dlf", + new String[] {paimonRestDlfAccessKey, + paimonRestDlfSecretKey}, + "DLF token provider requires 'paimon.rest.dlf.access-key-id' " + + "and 'paimon.rest.dlf.access-key-secret'"); + return rules; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java index f57bc87b6e15da..000bc75c529e12 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java @@ -25,6 +25,8 @@ import lombok.Setter; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; @@ -47,7 +49,7 @@ * determine if path-style URLs should be used for the storage system. */ public abstract class AbstractS3CompatibleProperties extends StorageProperties implements ObjectStorageProperties { - + private static final Logger LOG = LogManager.getLogger(AbstractS3CompatibleProperties.class); /** * The maximum number of concurrent connections that can be made to the object storage system. * This value is optional and can be configured by the user. @@ -172,16 +174,14 @@ public AwsCredentialsProvider getAwsCredentialsProvider() { return null; } - @Override public void initNormalizeAndCheckProps() { super.initNormalizeAndCheckProps(); - checkEndpoint(); - checkRequiredProperties(); - initRegionIfNecessary(); - if (StringUtils.isBlank(getRegion())) { - throw new IllegalArgumentException("region is required"); + setEndpointIfPossible(); + if (!isValidEndpoint(getEndpoint())) { + throw new IllegalArgumentException("Invalid endpoint: " + getEndpoint()); } + setRegionIfPossible(); } /** @@ -200,31 +200,48 @@ public void initNormalizeAndCheckProps() { * * @throws IllegalArgumentException if the endpoint format is invalid */ - protected void checkEndpoint() { - setEndpointIfNotSet(); - if (!isValidEndpoint(getEndpoint())) { - throw new IllegalArgumentException("Invalid endpoint format: " + getEndpoint()); + protected void setEndpointIfPossible() { + if (StringUtils.isNotBlank(getEndpoint())) { + return; + } + String endpoint = null; + // 1. try getting endpoint from uri + try { + endpoint = S3PropertyUtils.constructEndpointFromUrl(origProps, usePathStyle, forceParsingByStandardUrl); + } catch (Exception e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Failed to construct endpoint from url: " + origProps, e); + } + } + // 2. try getting endpoint region + if (StringUtils.isBlank(endpoint)) { + endpoint = getEndpointFromRegion(); + } + if (!StringUtils.isBlank(endpoint)) { + setEndpoint(endpoint); } } - private void initRegionIfNecessary() { + private void setRegionIfPossible() { if (StringUtils.isNotBlank(getRegion())) { return; } String endpoint = getEndpoint(); if (endpoint == null || endpoint.isEmpty()) { - throw new IllegalArgumentException("endpoint is required"); + return; } Optional regionOptional = extractRegion(endpoint); if (regionOptional.isPresent()) { setRegion(regionOptional.get()); - return; } - throw new IllegalArgumentException("Not a valid region, and cannot be parsed from endpoint: " + endpoint); } - public Optional extractRegion(String endpoint) { - for (Pattern pattern : endpointPatterns()) { + private Optional extractRegion(String endpoint) { + return extractRegion(endpointPatterns(), endpoint); + } + + public static Optional extractRegion(Set endpointPatterns, String endpoint) { + for (Pattern pattern : endpointPatterns) { Matcher matcher = pattern.matcher(endpoint.toLowerCase()); if (matcher.matches()) { // Check all possible groups for region (group 1, 2, or 3) @@ -243,7 +260,8 @@ public Optional extractRegion(String endpoint) { private boolean isValidEndpoint(String endpoint) { if (StringUtils.isBlank(endpoint)) { - return false; + // Endpoint is not required, so we consider it valid if empty. + return true; } for (Pattern pattern : endpointPatterns()) { Matcher matcher = pattern.matcher(endpoint.toLowerCase()); @@ -254,20 +272,6 @@ private boolean isValidEndpoint(String endpoint) { return false; } - private void setEndpointIfNotSet() { - if (StringUtils.isNotBlank(getEndpoint())) { - return; - } - String endpoint = S3PropertyUtils.constructEndpointFromUrl(origProps, usePathStyle, forceParsingByStandardUrl); - if (StringUtils.isBlank(endpoint)) { - endpoint = getEndpointFromRegion(); - } - if (StringUtils.isBlank(endpoint)) { - throw new IllegalArgumentException("endpoint is required"); - } - setEndpoint(endpoint); - } - // This method should be overridden by subclasses to provide a default endpoint based on the region. // Because for aws s3, only region is needed, the endpoint can be constructed from the region. // But for other s3 compatible storage, the endpoint may need to be specified explicitly. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java index 75f9fc8588196b..374807c48bd974 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java @@ -94,4 +94,11 @@ public static boolean guessIsMe(Map origProps) { protected Set endpointPatterns() { return ImmutableSet.of(Pattern.compile("^(?:https?://)?[a-zA-Z0-9.-]+(?::\\d+)?$")); } + + protected void setEndpointIfPossible() { + super.setEndpointIfPossible(); + if (StringUtils.isBlank(getEndpoint())) { + throw new IllegalArgumentException("Property minio.endpoint is required."); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java index 25f1bde582238f..4dda2977169c1e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java @@ -140,4 +140,11 @@ public void initializeHadoopStorageConfig() { hadoopStorageConfig.set("fs.obs.secret.key", secretKey); hadoopStorageConfig.set("fs.obs.endpoint", endpoint); } + + protected void setEndpointIfPossible() { + super.setEndpointIfPossible(); + if (StringUtils.isBlank(getEndpoint())) { + throw new IllegalArgumentException("Property obs.endpoint is required."); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java index 9031f4e4ef9ae5..45208e6f3d3a4b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java @@ -90,7 +90,7 @@ public class OSSProperties extends AbstractS3CompatibleProperties { * - s3.cn-hangzhou.aliyuncs.com => region = cn-hangzhou *

*/ - private static final Set ENDPOINT_PATTERN = ImmutableSet.of(Pattern + public static final Set ENDPOINT_PATTERN = ImmutableSet.of(Pattern .compile("^(?:https?://)?(?:s3\\.)?oss-([a-z0-9-]+?)(?:-internal)?\\.aliyuncs\\.com$"), Pattern.compile("(?:https?://)?([a-z]{2}-[a-z0-9-]+)\\.oss-dls\\.aliyuncs\\.com"), Pattern.compile("^(?:https?://)?dlf(?:-vpc)?\\.([a-z0-9-]+)\\.aliyuncs\\.com(?:/.*)?$")); @@ -120,6 +120,15 @@ protected static boolean guessIsMe(Map origProps) { return (value.contains("aliyuncs.com")); } + value = Stream.of("oss.region") + .map(origProps::get) + .filter(Objects::nonNull) + .findFirst() + .orElse(null); + if (StringUtils.isNotBlank(value)) { + return true; + } + Optional uriValue = origProps.entrySet().stream() .filter(e -> URI_KEYWORDS.stream() .anyMatch(key -> key.equalsIgnoreCase(e.getKey()))) @@ -147,7 +156,7 @@ private static boolean isKnownObjectStorage(String value) { } @Override - protected void checkEndpoint() { + protected void setEndpointIfPossible() { if (StringUtils.isBlank(this.endpoint) && StringUtils.isNotBlank(this.region)) { Optional uriValueOpt = origProps.entrySet().stream() .filter(e -> URI_KEYWORDS.stream() @@ -165,7 +174,7 @@ protected void checkEndpoint() { } } } - super.checkEndpoint(); + super.setEndpointIfPossible(); } @Override @@ -185,7 +194,6 @@ public void initNormalizeAndCheckProps() { // If only one is provided, it's an error throw new StoragePropertiesException( "Please set access_key and secret_key or omit both for anonymous access to public bucket."); - } private static String getOssEndpoint(String region, boolean publicAccess) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java index 0a4fda0bcdfe8f..bb66e66f5dcf81 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java @@ -22,12 +22,14 @@ import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException; import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.Map; import java.util.Optional; public class S3PropertyUtils { - + private static final Logger LOG = LogManager.getLogger(S3PropertyUtils.class); private static final String URI_KEY = "uri"; @@ -107,7 +109,6 @@ public static String constructRegionFromUrl(Map props, + " forceParsingByStandardUri: " + forceParsingByStandardUri, e); } return s3uri.getRegion().orElse(null); - } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java b/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java index 09860ee9a80550..238072df3c063e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java @@ -22,6 +22,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.util.S3URI; import org.apache.doris.common.util.S3Util; +import org.apache.doris.common.util.Util; import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties; import org.apache.doris.fs.remote.RemoteFile; @@ -257,11 +258,11 @@ public Status headObject(String remotePath) { return new Status(Status.ErrCode.NOT_FOUND, "remote path does not exist: " + remotePath); } else { LOG.warn("headObject failed:", e); - return new Status(Status.ErrCode.COMMON_ERROR, "headObject failed: " + e.getMessage()); + return new Status(Status.ErrCode.COMMON_ERROR, "headObject failed: " + Util.getRootCauseMessage(e)); } } catch (UserException ue) { LOG.warn("connect to s3 failed: ", ue); - return new Status(Status.ErrCode.COMMON_ERROR, "connect to s3 failed: " + ue.getMessage()); + return new Status(Status.ErrCode.COMMON_ERROR, "connect to s3 failed: " + Util.getRootCauseMessage(ue)); } } @@ -276,14 +277,17 @@ public Status getObject(String remoteFilePath, File localFile) { } return Status.OK; } catch (S3Exception s3Exception) { + LOG.warn("connect to s3 failed with s3 exception", s3Exception); return new Status( Status.ErrCode.COMMON_ERROR, - "get file from s3 error: " + s3Exception.awsErrorDetails().errorMessage()); + "get file from s3 error: " + s3Exception.awsErrorDetails().errorMessage() + + ". Root cause: " + Util.getRootCauseMessage(s3Exception)); } catch (UserException ue) { LOG.warn("connect to s3 failed: ", ue); - return new Status(Status.ErrCode.COMMON_ERROR, "connect to s3 failed: " + ue.getMessage()); + return new Status(Status.ErrCode.COMMON_ERROR, "connect to s3 failed: " + Util.getRootCauseMessage(ue)); } catch (Exception e) { - return new Status(Status.ErrCode.COMMON_ERROR, e.toString()); + LOG.warn("connect to s3 failed with unexpected exception", e); + return new Status(Status.ErrCode.COMMON_ERROR, Util.getRootCauseMessage(e)); } } @@ -303,10 +307,10 @@ public Status putObject(String remotePath, @Nullable InputStream content, long c return Status.OK; } catch (S3Exception e) { LOG.warn("put object failed: ", e); - return new Status(Status.ErrCode.COMMON_ERROR, "put object failed: " + e.getMessage()); + return new Status(Status.ErrCode.COMMON_ERROR, "put object failed: " + Util.getRootCauseMessage(e)); } catch (Exception ue) { LOG.warn("connect to s3 failed: ", ue); - return new Status(Status.ErrCode.COMMON_ERROR, "connect to s3 failed: " + ue.getMessage()); + return new Status(Status.ErrCode.COMMON_ERROR, "connect to s3 failed: " + Util.getRootCauseMessage(ue)); } } @@ -327,10 +331,10 @@ public Status deleteObject(String remotePath) { if (e.statusCode() == HttpStatus.SC_NOT_FOUND) { return Status.OK; } - return new Status(Status.ErrCode.COMMON_ERROR, "delete file failed: " + e.getMessage()); + return new Status(Status.ErrCode.COMMON_ERROR, "delete file failed: " + Util.getRootCauseMessage(e)); } catch (UserException ue) { LOG.warn("connect to s3 failed: ", ue); - return new Status(Status.ErrCode.COMMON_ERROR, "connect to s3 failed: " + ue.getMessage()); + return new Status(Status.ErrCode.COMMON_ERROR, "connect to s3 failed: " + Util.getRootCauseMessage(ue)); } } @@ -377,10 +381,11 @@ public Status deleteObjects(String absolutePath) { return Status.OK; } catch (DdlException e) { LOG.warn("deleteObjects:", e); - return new Status(Status.ErrCode.COMMON_ERROR, "list objects for delete objects failed: " + e.getMessage()); + return new Status(Status.ErrCode.COMMON_ERROR, + "list objects for delete objects failed: " + Util.getRootCauseMessage(e)); } catch (Exception e) { LOG.warn(String.format("delete objects %s failed", absolutePath), e); - return new Status(Status.ErrCode.COMMON_ERROR, "delete objects failed: " + e.getMessage()); + return new Status(Status.ErrCode.COMMON_ERROR, "delete objects failed: " + Util.getRootCauseMessage(e)); } } @@ -402,10 +407,10 @@ public Status copyObject(String origFilePath, String destFilePath) { return Status.OK; } catch (S3Exception e) { LOG.warn("copy file failed: ", e); - return new Status(Status.ErrCode.COMMON_ERROR, "copy file failed: " + e.getMessage()); + return new Status(Status.ErrCode.COMMON_ERROR, "copy file failed: " + Util.getRootCauseMessage(e)); } catch (UserException ue) { LOG.warn("copy to s3 failed: ", ue); - return new Status(Status.ErrCode.COMMON_ERROR, "connect to s3 failed: " + ue.getMessage()); + return new Status(Status.ErrCode.COMMON_ERROR, "connect to s3 failed: " + Util.getRootCauseMessage(ue)); } } @@ -430,7 +435,7 @@ public RemoteObjects listObjects(String absolutePath, String continuationToken) return new RemoteObjects(remoteObjects, response.isTruncated(), response.nextContinuationToken()); } catch (Exception e) { LOG.warn(String.format("Failed to list objects for S3: %s", absolutePath), e); - throw new DdlException("Failed to list objects for S3, Error message: " + e.getMessage(), e); + throw new DdlException("Failed to list objects for S3, Error message: " + Util.getRootCauseMessage(e), e); } } @@ -493,7 +498,7 @@ public Status multipartUpload(String remotePath, @Nullable InputStream inputStre } catch (Exception e) { LOG.warn("remotePath:{}, ", remotePath, e); st = new Status(Status.ErrCode.COMMON_ERROR, "Failed to multipartUpload " + remotePath - + " reason: " + e.getMessage()); + + " reason: " + Util.getRootCauseMessage(e)); if (uri != null && uploadId != null) { try { @@ -612,7 +617,8 @@ public Status globList(String remotePath, List result, boolean fileN return Status.OK; } catch (Exception e) { LOG.warn("Errors while getting file status", e); - return new Status(Status.ErrCode.COMMON_ERROR, "Errors while getting file status " + e.getMessage()); + return new Status(Status.ErrCode.COMMON_ERROR, + "Errors while getting file status " + Util.getRootCauseMessage(e)); } finally { long endTime = System.nanoTime(); long duration = endTime - startTime; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergVendedCredentialsProviderTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergVendedCredentialsProviderTest.java index cf46193f15bba4..57986853b1887c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergVendedCredentialsProviderTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergVendedCredentialsProviderTest.java @@ -17,6 +17,8 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.datasource.credentials.CredentialExtractor; +import org.apache.doris.datasource.credentials.CredentialUtils; import org.apache.doris.datasource.property.metastore.IcebergRestProperties; import org.apache.doris.datasource.property.metastore.MetastoreProperties; import org.apache.doris.datasource.property.storage.StorageProperties; @@ -36,8 +38,7 @@ public class IcebergVendedCredentialsProviderTest { @Test public void testS3CredentialExtractorWithValidCredentials() { - IcebergVendedCredentialsProvider.S3CredentialExtractor extractor = - new IcebergVendedCredentialsProvider.S3CredentialExtractor(); + IcebergS3CredentialExtractor extractor = new IcebergS3CredentialExtractor(); Map properties = new HashMap<>(); properties.put(S3FileIOProperties.ACCESS_KEY_ID, "test-access-key"); @@ -53,8 +54,7 @@ public void testS3CredentialExtractorWithValidCredentials() { @Test public void testS3CredentialExtractorWithPartialCredentials() { - IcebergVendedCredentialsProvider.S3CredentialExtractor extractor = - new IcebergVendedCredentialsProvider.S3CredentialExtractor(); + IcebergS3CredentialExtractor extractor = new IcebergS3CredentialExtractor(); Map properties = new HashMap<>(); properties.put(S3FileIOProperties.ACCESS_KEY_ID, "test-access-key"); @@ -70,8 +70,7 @@ public void testS3CredentialExtractorWithPartialCredentials() { @Test public void testS3CredentialExtractorWithEmptyProperties() { - IcebergVendedCredentialsProvider.S3CredentialExtractor extractor = - new IcebergVendedCredentialsProvider.S3CredentialExtractor(); + IcebergS3CredentialExtractor extractor = new IcebergS3CredentialExtractor(); Map credentials = extractor.extractCredentials(new HashMap<>()); Assertions.assertTrue(credentials.isEmpty()); @@ -82,8 +81,7 @@ public void testS3CredentialExtractorWithEmptyProperties() { @Test public void testS3CredentialExtractorWithNonAwsProperties() { - IcebergVendedCredentialsProvider.S3CredentialExtractor extractor = - new IcebergVendedCredentialsProvider.S3CredentialExtractor(); + IcebergS3CredentialExtractor extractor = new IcebergS3CredentialExtractor(); Map properties = new HashMap<>(); properties.put("some.other.property", "value"); @@ -134,10 +132,9 @@ public void testExtractCredentialsFromFileIO() { fileIoProperties.put(S3FileIOProperties.ACCESS_KEY_ID, "fileio-access-key"); fileIoProperties.put(S3FileIOProperties.SECRET_ACCESS_KEY, "fileio-secret-key"); - IcebergVendedCredentialsProvider.CredentialExtractor extractor = - new IcebergVendedCredentialsProvider.S3CredentialExtractor(); + IcebergS3CredentialExtractor extractor = new IcebergS3CredentialExtractor(); - Map credentials = IcebergVendedCredentialsProvider + Map credentials = CredentialUtils .extractCredentialsFromFileIO(fileIoProperties, extractor); Assertions.assertEquals("fileio-access-key", credentials.get("AWS_ACCESS_KEY")); @@ -147,8 +144,8 @@ public void testExtractCredentialsFromFileIO() { @Test public void testCustomCredentialExtractor() { - IcebergVendedCredentialsProvider.CredentialExtractor customExtractor = - new IcebergVendedCredentialsProvider.CredentialExtractor() { + CredentialExtractor customExtractor = + new CredentialExtractor() { @Override public Map extractCredentials(Map properties) { Map result = new HashMap<>(); @@ -162,7 +159,7 @@ public Map extractCredentials(Map properties) { Map properties = new HashMap<>(); properties.put("custom.key", "custom-value"); - Map credentials = IcebergVendedCredentialsProvider + Map credentials = CredentialUtils .extractCredentialsFromFileIO(properties, customExtractor); Assertions.assertEquals("custom-value", credentials.get("CUSTOM_BACKEND_PROPERTY")); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonVendedCredentialsProviderTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonVendedCredentialsProviderTest.java new file mode 100644 index 00000000000000..ec2bc64908a789 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonVendedCredentialsProviderTest.java @@ -0,0 +1,407 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.datasource.property.metastore.MetastoreProperties; +import org.apache.doris.datasource.property.metastore.PaimonRestMetaStoreProperties; +import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.datasource.property.storage.StorageProperties.Type; + +import org.apache.paimon.rest.RESTToken; +import org.apache.paimon.rest.RESTTokenFileIO; +import org.apache.paimon.table.Table; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.HashMap; +import java.util.Map; + +public class PaimonVendedCredentialsProviderTest { + + @Test + public void testOssCredentialExtractorWithValidCredentials() { + PaimonOssCredentialExtractor extractor = new PaimonOssCredentialExtractor(); + + Map properties = new HashMap<>(); + properties.put("fs.oss.accessKeyId", "test-access-key"); + properties.put("fs.oss.accessKeySecret", "test-secret-key"); + properties.put("fs.oss.securityToken", "test-session-token"); + properties.put("fs.oss.endpoint", "oss-cn-beijing.aliyuncs.com"); + + Map credentials = extractor.extractCredentials(properties); + + Assertions.assertEquals("test-access-key", credentials.get("AWS_ACCESS_KEY")); + Assertions.assertEquals("test-secret-key", credentials.get("AWS_SECRET_KEY")); + Assertions.assertEquals("test-session-token", credentials.get("AWS_TOKEN")); + Assertions.assertEquals("oss-cn-beijing.aliyuncs.com", credentials.get("AWS_ENDPOINT")); + Assertions.assertEquals("cn-beijing", credentials.get("AWS_REGION")); + } + + @Test + public void testOssCredentialExtractorWithPartialCredentials() { + PaimonOssCredentialExtractor extractor = new PaimonOssCredentialExtractor(); + + Map properties = new HashMap<>(); + properties.put("fs.oss.accessKeyId", "test-access-key"); + properties.put("fs.oss.accessKeySecret", "test-secret-key"); + // No session token and endpoint + + Map credentials = extractor.extractCredentials(properties); + + Assertions.assertEquals("test-access-key", credentials.get("AWS_ACCESS_KEY")); + Assertions.assertEquals("test-secret-key", credentials.get("AWS_SECRET_KEY")); + Assertions.assertFalse(credentials.containsKey("AWS_TOKEN")); + Assertions.assertFalse(credentials.containsKey("AWS_ENDPOINT")); + Assertions.assertFalse(credentials.containsKey("AWS_REGION")); + } + + @Test + public void testOssCredentialExtractorWithEmptyProperties() { + PaimonOssCredentialExtractor extractor = new PaimonOssCredentialExtractor(); + + Map credentials = extractor.extractCredentials(new HashMap<>()); + Assertions.assertTrue(credentials.isEmpty()); + + credentials = extractor.extractCredentials(null); + Assertions.assertTrue(credentials.isEmpty()); + } + + @Test + public void testOssCredentialExtractorWithNonOssProperties() { + PaimonOssCredentialExtractor extractor = new PaimonOssCredentialExtractor(); + + Map properties = new HashMap<>(); + properties.put("some.other.property", "value"); + properties.put("unrelated.key", "unrelated-value"); + + Map credentials = extractor.extractCredentials(properties); + Assertions.assertTrue(credentials.isEmpty()); + } + + @Test + public void testOssCredentialExtractorWithDifferentEndpoints() { + PaimonOssCredentialExtractor extractor = new PaimonOssCredentialExtractor(); + + // Test different OSS endpoint patterns + String[] endpoints = { + "oss-cn-shanghai.aliyuncs.com", + "oss-us-west-1.aliyuncs.com", + "oss-ap-southeast-1.aliyuncs.com" + }; + String[] expectedRegions = { + "cn-shanghai", + "us-west-1", + "ap-southeast-1" + }; + + for (int i = 0; i < endpoints.length; i++) { + Map properties = new HashMap<>(); + properties.put("fs.oss.accessKeyId", "test-access-key"); + properties.put("fs.oss.accessKeySecret", "test-secret-key"); + properties.put("fs.oss.endpoint", endpoints[i]); + + Map credentials = extractor.extractCredentials(properties); + + Assertions.assertEquals("test-access-key", credentials.get("AWS_ACCESS_KEY")); + Assertions.assertEquals("test-secret-key", credentials.get("AWS_SECRET_KEY")); + Assertions.assertEquals(endpoints[i], credentials.get("AWS_ENDPOINT")); + Assertions.assertEquals(expectedRegions[i], credentials.get("AWS_REGION")); + } + } + + @Test + public void testExtractVendedCredentialsFromTableWithDlfTokenProvider() { + Table table = Mockito.mock(Table.class); + RESTTokenFileIO restTokenFileIO = Mockito.mock(RESTTokenFileIO.class); + RESTToken restToken = Mockito.mock(RESTToken.class); + + Map tokenMap = new HashMap<>(); + tokenMap.put("fs.oss.accessKeyId", "dlf-access-key"); + tokenMap.put("fs.oss.accessKeySecret", "dlf-secret-key"); + tokenMap.put("fs.oss.securityToken", "dlf-session-token"); + tokenMap.put("fs.oss.endpoint", "oss-cn-beijing.aliyuncs.com"); + + Mockito.when(table.fileIO()).thenReturn(restTokenFileIO); + Mockito.when(table.name()).thenReturn("test_table"); + Mockito.when(restTokenFileIO.validToken()).thenReturn(restToken); + Mockito.when(restToken.token()).thenReturn(tokenMap); + + Map credentials = PaimonVendedCredentialsProvider + .extractVendedCredentialsFromTable("dlf", table); + + Assertions.assertEquals("dlf-access-key", credentials.get("AWS_ACCESS_KEY")); + Assertions.assertEquals("dlf-secret-key", credentials.get("AWS_SECRET_KEY")); + Assertions.assertEquals("dlf-session-token", credentials.get("AWS_TOKEN")); + Assertions.assertEquals("oss-cn-beijing.aliyuncs.com", credentials.get("AWS_ENDPOINT")); + Assertions.assertEquals("cn-beijing", credentials.get("AWS_REGION")); + } + + @Test + public void testExtractVendedCredentialsFromTableWithUnsupportedTokenProvider() { + Table table = Mockito.mock(Table.class); + RESTTokenFileIO restTokenFileIO = Mockito.mock(RESTTokenFileIO.class); + RESTToken restToken = Mockito.mock(RESTToken.class); + + Map tokenMap = new HashMap<>(); + tokenMap.put("fs.oss.accessKeyId", "unsupported-access-key"); + tokenMap.put("fs.oss.accessKeySecret", "unsupported-secret-key"); + + Mockito.when(table.fileIO()).thenReturn(restTokenFileIO); + Mockito.when(table.name()).thenReturn("test_table"); + Mockito.when(restTokenFileIO.validToken()).thenReturn(restToken); + Mockito.when(restToken.token()).thenReturn(tokenMap); + + Map credentials = PaimonVendedCredentialsProvider + .extractVendedCredentialsFromTable("unsupported", table); + + Assertions.assertTrue(credentials.isEmpty()); + } + + @Test + public void testExtractVendedCredentialsFromTableWithNullTable() { + Map credentials = PaimonVendedCredentialsProvider + .extractVendedCredentialsFromTable("dlf", null); + Assertions.assertTrue(credentials.isEmpty()); + } + + @Test + public void testExtractVendedCredentialsFromTableWithNullFileIO() { + Table table = Mockito.mock(Table.class); + Mockito.when(table.fileIO()).thenReturn(null); + + Map credentials = PaimonVendedCredentialsProvider + .extractVendedCredentialsFromTable("dlf", table); + Assertions.assertTrue(credentials.isEmpty()); + } + + @Test + public void testExtractVendedCredentialsFromTableWithNonRESTTokenFileIO() { + Table table = Mockito.mock(Table.class); + // fileIO returns null, which is not an instance of RESTTokenFileIO + Mockito.when(table.fileIO()).thenReturn(null); + Mockito.when(table.name()).thenReturn("test_table"); + + Map credentials = PaimonVendedCredentialsProvider + .extractVendedCredentialsFromTable("dlf", table); + + Assertions.assertTrue(credentials.isEmpty()); + } + + @Test + public void testGetBackendLocationPropertiesWithPaimonRestMetaStore() { + // Mock PaimonRestMetaStoreProperties + PaimonRestMetaStoreProperties restProperties = Mockito.mock(PaimonRestMetaStoreProperties.class); + Mockito.when(restProperties.getTokenProvider()).thenReturn("dlf"); + + // Mock storage properties + StorageProperties storageProperties = Mockito.mock(StorageProperties.class); + Map baseProperties = new HashMap<>(); + baseProperties.put("base.property", "base-value"); + + Map vendedProperties = new HashMap<>(); + vendedProperties.put("base.property", "base-value"); + vendedProperties.put("AWS_ACCESS_KEY", "vended-access-key"); + vendedProperties.put("AWS_SECRET_KEY", "vended-secret-key"); + vendedProperties.put("AWS_TOKEN", "vended-session-token"); + + Mockito.when(storageProperties.getBackendConfigProperties()).thenReturn(baseProperties); + Mockito.when(storageProperties.getBackendConfigProperties(Mockito.anyMap())).thenReturn(vendedProperties); + + Map storagePropertiesMap = new HashMap<>(); + storagePropertiesMap.put(Type.S3, storageProperties); + + // Mock table with vended credentials + Table table = Mockito.mock(Table.class); + RESTTokenFileIO restTokenFileIO = Mockito.mock(RESTTokenFileIO.class); + RESTToken restToken = Mockito.mock(RESTToken.class); + + Map tokenMap = new HashMap<>(); + tokenMap.put("fs.oss.accessKeyId", "vended-access-key"); + tokenMap.put("fs.oss.accessKeySecret", "vended-secret-key"); + tokenMap.put("fs.oss.securityToken", "vended-session-token"); + + Mockito.when(table.fileIO()).thenReturn(restTokenFileIO); + Mockito.when(table.name()).thenReturn("test_table"); + Mockito.when(restTokenFileIO.validToken()).thenReturn(restToken); + Mockito.when(restToken.token()).thenReturn(tokenMap); + + // Test with PaimonRestMetaStoreProperties + Map result = PaimonVendedCredentialsProvider.getBackendLocationProperties( + restProperties, storagePropertiesMap, table); + + Assertions.assertEquals("base-value", result.get("base.property")); + Assertions.assertEquals("vended-access-key", result.get("AWS_ACCESS_KEY")); + Assertions.assertEquals("vended-secret-key", result.get("AWS_SECRET_KEY")); + Assertions.assertEquals("vended-session-token", result.get("AWS_TOKEN")); + } + + @Test + public void testGetBackendLocationPropertiesWithNonPaimonRestMetaStore() { + // Mock non-PaimonRest metastore properties + MetastoreProperties metastoreProperties = Mockito.mock(MetastoreProperties.class); + + // Mock storage properties + StorageProperties storageProperties = Mockito.mock(StorageProperties.class); + Map baseProperties = new HashMap<>(); + baseProperties.put("base.property", "base-value"); + baseProperties.put("AWS_ACCESS_KEY", "static-access-key"); + + Mockito.when(storageProperties.getBackendConfigProperties(Mockito.anyMap())).thenReturn(baseProperties); + + Map storagePropertiesMap = new HashMap<>(); + storagePropertiesMap.put(Type.S3, storageProperties); + + // Mock table (should be ignored for non-REST catalogs) + Table table = Mockito.mock(Table.class); + + // Test with non-PaimonRest metastore + Map result = PaimonVendedCredentialsProvider.getBackendLocationProperties( + metastoreProperties, storagePropertiesMap, table); + + Assertions.assertEquals("base-value", result.get("base.property")); + Assertions.assertEquals("static-access-key", result.get("AWS_ACCESS_KEY")); + Assertions.assertFalse(result.containsKey("AWS_TOKEN")); + } + + @Test + public void testGetBackendLocationPropertiesWithMultipleStorageTypes() { + // Mock PaimonRestMetaStoreProperties + PaimonRestMetaStoreProperties restProperties = Mockito.mock(PaimonRestMetaStoreProperties.class); + Mockito.when(restProperties.getTokenProvider()).thenReturn("dlf"); + + // Mock S3 storage properties + StorageProperties s3Properties = Mockito.mock(StorageProperties.class); + Map s3VendedProperties = new HashMap<>(); + s3VendedProperties.put("s3.property", "s3-value"); + s3VendedProperties.put("AWS_ACCESS_KEY", "vended-access-key"); + + Mockito.when(s3Properties.getBackendConfigProperties(Mockito.anyMap())).thenReturn(s3VendedProperties); + + // Mock HDFS storage properties + StorageProperties hdfsStorageProperties = Mockito.mock(StorageProperties.class); + Map hdfsPropertiesMap = new HashMap<>(); + hdfsPropertiesMap.put("hdfs.property", "hdfs-value"); + + Mockito.when(hdfsStorageProperties.getBackendConfigProperties(Mockito.anyMap())).thenReturn(hdfsPropertiesMap); + + Map storagePropertiesMap = new HashMap<>(); + storagePropertiesMap.put(Type.S3, s3Properties); + storagePropertiesMap.put(Type.HDFS, hdfsStorageProperties); + + // Mock table with vended credentials + Table table = Mockito.mock(Table.class); + RESTTokenFileIO restTokenFileIO = Mockito.mock(RESTTokenFileIO.class); + RESTToken restToken = Mockito.mock(RESTToken.class); + + Map tokenMap = new HashMap<>(); + tokenMap.put("fs.oss.accessKeyId", "vended-access-key"); + + Mockito.when(table.fileIO()).thenReturn(restTokenFileIO); + Mockito.when(table.name()).thenReturn("test_table"); + Mockito.when(restTokenFileIO.validToken()).thenReturn(restToken); + Mockito.when(restToken.token()).thenReturn(tokenMap); + + // Test with multiple storage types + Map result = PaimonVendedCredentialsProvider.getBackendLocationProperties( + restProperties, storagePropertiesMap, table); + + Assertions.assertEquals("s3-value", result.get("s3.property")); + Assertions.assertEquals("hdfs-value", result.get("hdfs.property")); + Assertions.assertEquals("vended-access-key", result.get("AWS_ACCESS_KEY")); + } + + @Test + public void testGetBackendLocationPropertiesWithNullTable() { + // Mock PaimonRestMetaStoreProperties + PaimonRestMetaStoreProperties restProperties = Mockito.mock(PaimonRestMetaStoreProperties.class); + Mockito.when(restProperties.getTokenProvider()).thenReturn("dlf"); + + // Mock storage properties + StorageProperties storageProperties = Mockito.mock(StorageProperties.class); + Map baseProperties = new HashMap<>(); + baseProperties.put("base.property", "base-value"); + + Mockito.when(storageProperties.getBackendConfigProperties(Mockito.anyMap())).thenReturn(baseProperties); + + Map storagePropertiesMap = new HashMap<>(); + storagePropertiesMap.put(Type.S3, storageProperties); + + // Test with null table (should fall back to base properties without vended credentials) + Map result = PaimonVendedCredentialsProvider.getBackendLocationProperties( + restProperties, storagePropertiesMap, null); + + Assertions.assertEquals("base-value", result.get("base.property")); + Assertions.assertFalse(result.containsKey("AWS_ACCESS_KEY")); + } + + @Test + public void testGetBackendLocationPropertiesWithEmptyStorageMap() { + // Mock PaimonRestMetaStoreProperties + PaimonRestMetaStoreProperties restProperties = Mockito.mock(PaimonRestMetaStoreProperties.class); + Mockito.when(restProperties.getTokenProvider()).thenReturn("dlf"); + + // Empty storage properties map + Map storagePropertiesMap = new HashMap<>(); + + // Mock table + Table table = Mockito.mock(Table.class); + + // Test with empty storage map + Map result = PaimonVendedCredentialsProvider.getBackendLocationProperties( + restProperties, storagePropertiesMap, table); + + Assertions.assertTrue(result.isEmpty()); + } + + @Test + public void testGetBackendLocationPropertiesWithEmptyTokenMap() { + // Mock PaimonRestMetaStoreProperties + PaimonRestMetaStoreProperties restProperties = Mockito.mock(PaimonRestMetaStoreProperties.class); + Mockito.when(restProperties.getTokenProvider()).thenReturn("dlf"); + + // Mock storage properties + StorageProperties storageProperties = Mockito.mock(StorageProperties.class); + Map baseProperties = new HashMap<>(); + baseProperties.put("base.property", "base-value"); + + Mockito.when(storageProperties.getBackendConfigProperties(Mockito.anyMap())).thenReturn(baseProperties); + + Map storagePropertiesMap = new HashMap<>(); + storagePropertiesMap.put(Type.S3, storageProperties); + + // Mock table with empty token map + Table table = Mockito.mock(Table.class); + RESTTokenFileIO restTokenFileIO = Mockito.mock(RESTTokenFileIO.class); + RESTToken restToken = Mockito.mock(RESTToken.class); + + Map emptyTokenMap = new HashMap<>(); + + Mockito.when(table.fileIO()).thenReturn(restTokenFileIO); + Mockito.when(table.name()).thenReturn("test_table"); + Mockito.when(restTokenFileIO.validToken()).thenReturn(restToken); + Mockito.when(restToken.token()).thenReturn(emptyTokenMap); + + // Test with empty token map (should fall back to base properties without vended credentials) + Map result = PaimonVendedCredentialsProvider.getBackendLocationProperties( + restProperties, storagePropertiesMap, table); + + Assertions.assertEquals("base-value", result.get("base.property")); + Assertions.assertFalse(result.containsKey("AWS_ACCESS_KEY")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/RestCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergGlueRestCatalogTest.java similarity index 88% rename from fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/RestCatalogTest.java rename to fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergGlueRestCatalogTest.java index dca4f91c789223..7230b235ddfc78 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/RestCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergGlueRestCatalogTest.java @@ -30,28 +30,53 @@ import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.io.CloseableIterable; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.util.Map; -@Disabled("Disabled until fill ak/sk") -public class RestCatalogTest { +@Disabled("set your AWS access key, secret key and account ID before running the test") +public class IcebergGlueRestCatalogTest { - // set your AWS access key and secret key - private String ak = ""; - private String sk = ""; + private String awsAk = ""; + private String awsSk = ""; + private String awsAccountId = ""; - @BeforeEach - protected void initCatalog() { + @Test + public void testIcebergGlueRestCatalog() { + Catalog glueRestCatalog = initIcebergGlueRestCatalog(); + SupportsNamespaces nsCatalog = (SupportsNamespaces) glueRestCatalog; + // List namespaces and assert + nsCatalog.listNamespaces(Namespace.empty()).forEach(namespace1 -> { + System.out.println("Namespace: " + namespace1); + Assertions.assertNotNull(namespace1, "Namespace should not be null"); + + glueRestCatalog.listTables(namespace1).forEach(tableIdentifier -> { + System.out.println("Table: " + tableIdentifier.name()); + Assertions.assertNotNull(tableIdentifier, "TableIdentifier should not be null"); + + // Load table history and assert + Table iceTable = glueRestCatalog.loadTable(tableIdentifier); + iceTable.history().forEach(snapshot -> { + System.out.println("Snapshot: " + snapshot); + Assertions.assertNotNull(snapshot, "Snapshot should not be null"); + }); + + CloseableIterable tasks = iceTable.newScan().planFiles(); + tasks.forEach(task -> { + System.out.println("FileScanTask: " + task); + Assertions.assertNotNull(task, "FileScanTask should not be null"); + }); + }); + }); } - private Catalog initGlueRestCatalog() { + private Catalog initIcebergGlueRestCatalog() { Map options = Maps.newHashMap(); options.put(CatalogUtil.ICEBERG_CATALOG_TYPE, CatalogUtil.ICEBERG_CATALOG_TYPE_REST); options.put(CatalogProperties.URI, "https://glue.ap-east-1.amazonaws.com/iceberg"); - options.put(CatalogProperties.WAREHOUSE_LOCATION, "169698404049:s3tablescatalog/s3-table-bucket-hk-glue-test"); + options.put(CatalogProperties.WAREHOUSE_LOCATION, + awsAccountId + ":s3tablescatalog/s3-table-bucket-hk-glue-test"); // remove this endpoint prop, or, add https:// options.put(S3FileIOProperties.ENDPOINT, "https://s3.ap-east-1.amazonaws.com"); // must set: @@ -69,10 +94,10 @@ private Catalog initGlueRestCatalog() { // Forbidden: {"message":"Credential should be scoped to correct service: 'glue'. "} options.put("rest.signing-name", "glue"); // Forbidden: {"message":"The security token included in the request is invalid."} - options.put("rest.access-key-id", ak); + options.put("rest.access-key-id", awsAk); // Forbidden: {"message":"The request signature we calculated does not match the signature you provided. // Check your AWS Secret Access Key and signing method. Consult the service documentation for details."} - options.put("rest.secret-access-key", sk); + options.put("rest.secret-access-key", awsSk); // same as AwsClientProperties.CLIENT_REGION, "ap-east-1" options.put("rest.signing-region", "ap-east-1"); // options.put("iceberg.catalog.warehouse", ":s3tablescatalog/"); @@ -80,33 +105,4 @@ private Catalog initGlueRestCatalog() { Configuration conf = new Configuration(); return CatalogUtil.buildIcebergCatalog("glue_test", options, conf); } - - @Test - public void testGlueRestCatalog() { - Catalog glueRestCatalog = initGlueRestCatalog(); - SupportsNamespaces nsCatalog = (SupportsNamespaces) glueRestCatalog; - // List namespaces and assert - nsCatalog.listNamespaces(Namespace.empty()).forEach(namespace1 -> { - System.out.println("Namespace: " + namespace1); - Assertions.assertNotNull(namespace1, "Namespace should not be null"); - - glueRestCatalog.listTables(namespace1).forEach(tableIdentifier -> { - System.out.println("Table: " + tableIdentifier.name()); - Assertions.assertNotNull(tableIdentifier, "TableIdentifier should not be null"); - - // Load table history and assert - Table iceTable = glueRestCatalog.loadTable(tableIdentifier); - iceTable.history().forEach(snapshot -> { - System.out.println("Snapshot: " + snapshot); - Assertions.assertNotNull(snapshot, "Snapshot should not be null"); - }); - - CloseableIterable tasks = iceTable.newScan().planFiles(); - tasks.forEach(task -> { - System.out.println("FileScanTask: " + task); - Assertions.assertNotNull(task, "FileScanTask should not be null"); - }); - }); - }); - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonDlfRestCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonDlfRestCatalogTest.java new file mode 100644 index 00000000000000..99a7f931e3ddc8 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonDlfRestCatalogTest.java @@ -0,0 +1,285 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.metastore; + +import org.apache.doris.backup.Status; +import org.apache.doris.common.UserException; +import org.apache.doris.common.util.S3URI; +import org.apache.doris.common.util.Util; +import org.apache.doris.datasource.property.storage.S3Properties; +import org.apache.doris.fs.StorageTypeMapper; +import org.apache.doris.fs.remote.S3FileSystem; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicSessionCredentials; +import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.S3Object; +import com.google.common.collect.Maps; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.paimon.catalog.Catalog.DatabaseNotExistException; +import org.apache.paimon.catalog.Catalog.TableNotExistException; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; +import org.apache.paimon.catalog.Database; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.options.Options; +import org.apache.paimon.rest.RESTToken; +import org.apache.paimon.rest.RESTTokenFileIO; +import org.apache.paimon.table.source.DataSplit; +import org.apache.paimon.table.source.RawFile; +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.table.source.Split; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3Configuration; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +@Disabled("set aliyun access key, secret key before running the test") +public class PaimonDlfRestCatalogTest { + + private String aliyunAk = ""; + private String aliyunSk = ""; + + @Test + public void testPaimonDlfRestCatalog() throws DatabaseNotExistException, TableNotExistException, UserException { + org.apache.paimon.catalog.Catalog catalog = initPaimonDlfRestCatalog(); + System.out.println(catalog); + List dbs = catalog.listDatabases(); + for (String dbName : dbs) { + System.out.println("test debug get db: " + dbName); + Database db = catalog.getDatabase(dbName); + System.out.println("test debug get db instance: " + db.name() + ", " + db.options() + ", " + db.comment()); + List tables = catalog.listTables(dbName); + for (String tblName : tables) { + System.out.println("test debug get table: " + tblName); + if (!tblName.equalsIgnoreCase("users_samples")) { + continue; + } + org.apache.paimon.table.Table table = catalog.getTable( + org.apache.paimon.catalog.Identifier.create(dbName, tblName)); + System.out.println("test debug get table instance: " + table.name() + ", " + table.options() + ", " + + table.comment()); + + FileIO fileIO = table.fileIO(); + if (fileIO instanceof RESTTokenFileIO) { + System.out.println("test debug get file io instance: " + fileIO.getClass().getName()); + RESTTokenFileIO restTokenFileIO = (RESTTokenFileIO) fileIO; + RESTToken restToken = restTokenFileIO.validToken(); + Map tokens = restToken.token(); + for (Map.Entry kv : tokens.entrySet()) { + System.out.println("test debug get token: " + kv.getKey() + ", " + kv.getValue()); + } + // String accType = tokens.get("fs.oss.token.access.type"); + String tmpAk = tokens.get("fs.oss.accessKeyId"); + String tmpSk = tokens.get("fs.oss.accessKeySecret"); + String stsToken = tokens.get("fs.oss.securityToken"); + String endpoint = tokens.get("fs.oss.endpoint"); + + ReadBuilder readBuilder = table.newReadBuilder(); + List paimonSplits = readBuilder.newScan().plan().splits(); + for (Split split : paimonSplits) { + System.out.println("test debug get split: " + split); + if (split instanceof DataSplit) { + DataSplit dataSplit = (DataSplit) split; + Optional> rawFiles = dataSplit.convertToRawFiles(); + if (rawFiles.isPresent()) { + for (RawFile rawFile : rawFiles.get()) { + System.out.println("test debug get raw file: " + rawFile.path()); + readByDorisS3FileSystem(rawFile.path(), tmpAk, tmpSk, stsToken, endpoint, + "oss-cn-beijing"); + readByAwsSdkV1(rawFile.path(), tmpAk, tmpSk, stsToken, endpoint, "oss-cn-beijing"); + readByAwsSdkV2(rawFile.path(), tmpAk, tmpSk, stsToken, endpoint, "oss-cn-beijing"); + } + } else { + System.out.println("test debug no raw files in this data split"); + } + } + } + } else { + System.out.println( + "test debug fileIO is not RESTTokenFileIO, it is: " + fileIO.getClass().getName()); + } + } + } + } + + /** + * https://paimon.apache.org/docs/1.1/concepts/rest/dlf/ + * CREATE CATALOG `paimon-rest-catalog` + * WITH ( + * 'type' = 'paimon', + * 'uri' = '', + * 'metastore' = 'rest', + * 'warehouse' = 'my_instance_name', + * 'token.provider' = 'dlf', + * 'dlf.access-key-id'='', + * 'dlf.access-key-secret'='', + * ); + * + * @return + */ + private org.apache.paimon.catalog.Catalog initPaimonDlfRestCatalog() { + HiveConf hiveConf = new HiveConf(); + Options catalogOptions = new Options(); + catalogOptions.set("metastore", "rest"); + catalogOptions.set("warehouse", "new_dfl_paimon_catalog"); + catalogOptions.set("uri", "http://cn-beijing-vpc.dlf.aliyuncs.com"); + catalogOptions.set("token.provider", "dlf"); + catalogOptions.set("dlf.access-key-id", aliyunAk); + catalogOptions.set("dlf.access-key-secret", aliyunSk); + CatalogContext catalogContext = CatalogContext.create(catalogOptions, hiveConf); + return CatalogFactory.createCatalog(catalogContext); + } + + private void readByDorisS3FileSystem(String path, String tmpAk, String tmpSk, String stsToken, String endpoint, + String region) { + // replace "oss://" with "s3://" + String finalPath = path.startsWith("oss://") ? path.replace("oss://", "s3://") : path; + System.out.println("test debug final path: " + finalPath); + Map props = Maps.newHashMap(); + props.put("s3.endpoint", endpoint); + props.put("s3.region", region); + props.put("s3.access_key", tmpAk); + props.put("s3.secret_key", tmpSk); + props.put("s3.session_token", stsToken); + S3Properties s3Properties = S3Properties.of(props); + S3FileSystem s3Fs = (S3FileSystem) StorageTypeMapper.create(s3Properties); + File localFile = new File("/tmp/s3/" + System.currentTimeMillis() + ".data"); + if (localFile.exists()) { + try { + Files.delete(localFile.toPath()); + } catch (IOException e) { + System.err.println("Failed to delete existing local file: " + localFile.getAbsolutePath()); + e.printStackTrace(); + } + } else { + localFile.getParentFile().mkdirs(); // Ensure parent directories exist + } + Status st = s3Fs.getObjStorage().getObject(finalPath, localFile); + System.out.println(st); + if (st.ok()) { + System.out.println("test debug local path: " + localFile.getAbsolutePath()); + } else { + Assertions.fail(st.toString()); + } + } + + private void readByAwsSdkV1(String filePath, String accessKeyId, String secretAccessKey, + String sessionToken, String endpoint, String region) throws UserException { + BasicSessionCredentials sessionCredentials = new BasicSessionCredentials( + accessKeyId, + secretAccessKey, + sessionToken + ); + ClientConfiguration clientConfig = new ClientConfiguration(); + clientConfig.setSignerOverride("AWSS3V4SignerType"); + + AmazonS3 s3Client = AmazonS3ClientBuilder.standard() + .withCredentials(new AWSStaticCredentialsProvider(sessionCredentials)) + .withEndpointConfiguration(new EndpointConfiguration(endpoint, region)) + .withClientConfiguration(clientConfig) + .withPathStyleAccessEnabled(false) + .build(); + + S3URI s3URI = S3URI.create(filePath); + System.out.println("test debug s3uri: " + s3URI); + try { + String content = downloadAndReadFileWithSdkV1(s3Client, s3URI.getBucket(), s3URI.getKey()); + System.out.println("Content: " + content); + } catch (Exception e) { + e.printStackTrace(); + Assertions.fail(Util.getRootCauseMessage(e)); + } + } + + private String downloadAndReadFileWithSdkV1(AmazonS3 s3Client, String bucketName, String objectKey) + throws IOException { + S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucketName, objectKey)); + try (InputStream inputStream = s3Object.getObjectContent(); + InputStreamReader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8)) { + StringBuilder content = new StringBuilder(); + char[] buffer = new char[1024]; + int bytesRead; + while ((bytesRead = reader.read(buffer)) != -1) { + content.append(buffer, 0, bytesRead); + } + return content.toString(); + } + } + + private void readByAwsSdkV2(String path, String tmpAk, String tmpSk, String stsToken, String endpoint, + String region) { + S3Client s3Client = S3Client.builder() + .credentialsProvider(StaticCredentialsProvider.create(AwsSessionCredentials.create(tmpAk, tmpSk, + stsToken))) + .region(Region.of(region)) + .endpointOverride(URI.create("https://" + endpoint)) + .serviceConfiguration(S3Configuration.builder() + .chunkedEncodingEnabled(false) + .pathStyleAccessEnabled(false) + .build()) + .build(); + try { + S3URI s3URI = S3URI.create(path); + System.out.println("test debug s3uri: " + s3URI); + downloadAndReadFileWithSdkV2(s3Client, s3URI.getBucket(), s3URI.getKey()); + } catch (Exception e) { + Assertions.fail(Util.getRootCauseMessage(e)); + } finally { + s3Client.close(); + } + } + + private void downloadAndReadFileWithSdkV2(S3Client s3Client, String bucketName, String objectKey) + throws IOException { + software.amazon.awssdk.services.s3.model.GetObjectRequest request + = software.amazon.awssdk.services.s3.model.GetObjectRequest.builder() + .bucket(bucketName) + .key(objectKey) + .build(); + + try (ResponseInputStream inputStream = s3Client.getObject(request); + BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) { + String line; + while ((line = reader.readLine()) != null) { + System.out.println(line); + } + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonRestMetaStorePropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonRestMetaStorePropertiesTest.java new file mode 100644 index 00000000000000..265e67c3b6528f --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonRestMetaStorePropertiesTest.java @@ -0,0 +1,400 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.metastore; + +import org.apache.doris.datasource.paimon.PaimonExternalCatalog; +import org.apache.doris.datasource.property.storage.StorageProperties; + +import org.apache.paimon.options.Options; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class PaimonRestMetaStorePropertiesTest { + + @Test + public void testBasicRestProperties() { + Map props = new HashMap<>(); + props.put("paimon.rest.uri", "http://localhost:8080"); + props.put("warehouse", "catalog_name"); + props.put("paimon.rest.token.provider", "none"); + + PaimonRestMetaStoreProperties restProps = new PaimonRestMetaStoreProperties(props); + restProps.initNormalizeAndCheckProps(); + + Assertions.assertEquals(PaimonExternalCatalog.PAIMON_REST, restProps.getPaimonCatalogType()); + Assertions.assertEquals("rest", restProps.getMetastoreType()); + } + + @Test + public void testUriAliases() { + // Test different URI property names + Map props1 = new HashMap<>(); + props1.put("uri", "http://localhost:8080"); + props1.put("paimon.rest.token.provider", "none"); + props1.put("warehouse", "catalog_name"); + PaimonRestMetaStoreProperties restProps1 = new PaimonRestMetaStoreProperties(props1); + restProps1.initNormalizeAndCheckProps(); + + Map props2 = new HashMap<>(); + props2.put("paimon.rest.uri", "http://localhost:8080"); + props2.put("paimon.rest.token.provider", "none"); + props2.put("warehouse", "catalog_name"); + PaimonRestMetaStoreProperties restProps2 = new PaimonRestMetaStoreProperties(props2); + restProps2.initNormalizeAndCheckProps(); + + // Both should work and set the same URI in catalog options + List storagePropertiesList = new ArrayList<>(); + restProps1.buildCatalogOptions(storagePropertiesList); + restProps2.buildCatalogOptions(storagePropertiesList); + + Options options1 = restProps1.getCatalogOptions(); + Options options2 = restProps2.getCatalogOptions(); + + Assertions.assertEquals("http://localhost:8080", options1.get("uri")); + Assertions.assertEquals("http://localhost:8080", options2.get("uri")); + } + + @Test + public void testPaimonRestPropertiesPassthrough() { + Map props = new HashMap<>(); + props.put("paimon.rest.uri", "http://localhost:8080"); + props.put("paimon.rest.custom.property", "custom-value"); + props.put("paimon.rest.timeout", "30000"); + props.put("paimon.rest.retry.count", "3"); + props.put("paimon.rest.token.provider", "none"); + props.put("warehouse", "catalog_name"); + + PaimonRestMetaStoreProperties restProps = new PaimonRestMetaStoreProperties(props); + restProps.initNormalizeAndCheckProps(); + + List storagePropertiesList = new ArrayList<>(); + restProps.buildCatalogOptions(storagePropertiesList); + Options catalogOptions = restProps.getCatalogOptions(); + + // Basic URI should be set + Assertions.assertEquals("http://localhost:8080", catalogOptions.get("uri")); + + // Custom paimon.rest.* properties should be passed through without prefix + Assertions.assertEquals("custom-value", catalogOptions.get("custom.property")); + Assertions.assertEquals("30000", catalogOptions.get("timeout")); + Assertions.assertEquals("3", catalogOptions.get("retry.count")); + } + + @Test + public void testTokenProviderProperty() { + Map props = new HashMap<>(); + props.put("paimon.rest.uri", "http://localhost:8080"); + props.put("paimon.rest.token.provider", "dlf"); + props.put("warehouse", "catalog_name"); + props.put("paimon.rest.dlf.access-key-id", "ak"); + props.put("paimon.rest.dlf.access-key-secret", "sk"); + + PaimonRestMetaStoreProperties restProps = new PaimonRestMetaStoreProperties(props); + restProps.initNormalizeAndCheckProps(); + + Assertions.assertEquals("dlf", restProps.getTokenProvider()); + } + + @Test + public void testDlfTokenProviderValidConfiguration() { + Map props = new HashMap<>(); + props.put("paimon.rest.uri", "http://localhost:8080"); + props.put("paimon.rest.token.provider", "dlf"); + props.put("paimon.rest.dlf.access-key-id", "ak"); + props.put("paimon.rest.dlf.access-key-secret", "sk"); + props.put("warehouse", "catalog_name"); + + PaimonRestMetaStoreProperties restProps = new PaimonRestMetaStoreProperties(props); + restProps.initNormalizeAndCheckProps(); // Should not throw + + Assertions.assertEquals("dlf", restProps.getTokenProvider()); + } + + @Test + public void testDlfTokenProviderMissingAccessKeyId() { + Map props = new HashMap<>(); + props.put("paimon.rest.uri", "http://localhost:8080"); + props.put("paimon.rest.token.provider", "dlf"); + props.put("paimon.rest.dlf.access-key-secret", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); + // Missing access-key-id + + PaimonRestMetaStoreProperties restProps = new PaimonRestMetaStoreProperties(props); + Assertions.assertThrows(IllegalArgumentException.class, restProps::initNormalizeAndCheckProps); + } + + @Test + public void testDlfTokenProviderMissingSecretKey() { + Map props = new HashMap<>(); + props.put("paimon.rest.uri", "http://localhost:8080"); + props.put("paimon.rest.token.provider", "dlf"); + props.put("paimon.rest.dlf.access-key-id", "AKIAIOSFODNN7EXAMPLE"); + // Missing secret-access-key + + PaimonRestMetaStoreProperties restProps = new PaimonRestMetaStoreProperties(props); + Assertions.assertThrows(IllegalArgumentException.class, restProps::initNormalizeAndCheckProps); + } + + @Test + public void testDlfTokenProviderMissingBothCredentials() { + Map props = new HashMap<>(); + props.put("paimon.rest.uri", "http://localhost:8080"); + props.put("paimon.rest.token.provider", "dlf"); + props.put("warehouse", "catalog_name"); + // Missing both credentials + + PaimonRestMetaStoreProperties restProps = new PaimonRestMetaStoreProperties(props); + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, restProps::initNormalizeAndCheckProps); + + // The error message should mention the required properties + String errorMessage = exception.getMessage(); + Assertions.assertTrue(errorMessage.contains("access-key-id") + && errorMessage.contains("access-key-secret")); + } + + @Test + public void testNonDlfTokenProviderDoesNotRequireCredentials() { + // Test that non-dlf token providers don't require DLF credentials + Map props = new HashMap<>(); + props.put("paimon.rest.uri", "http://localhost:8080"); + props.put("paimon.rest.token.provider", "custom-provider"); + props.put("warehouse", "catalog_name"); + + PaimonRestMetaStoreProperties restProps = new PaimonRestMetaStoreProperties(props); + restProps.initNormalizeAndCheckProps(); // Should not throw + + Assertions.assertEquals("custom-provider", restProps.getTokenProvider()); + } + + @Test + public void testNonDlfTokenProviderWithDlfCredentialsStillWorks() { + // Test that non-dlf token provider doesn't require DLF credentials + Map props = new HashMap<>(); + props.put("paimon.rest.uri", "http://localhost:8080"); + props.put("paimon.rest.token.provider", "other"); + props.put("paimon.rest.dlf.access-key-id", "AKIAIOSFODNN7EXAMPLE"); + props.put("paimon.rest.dlf.access-key-secret", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); + props.put("warehouse", "catalog_name"); + + PaimonRestMetaStoreProperties restProps = new PaimonRestMetaStoreProperties(props); + restProps.initNormalizeAndCheckProps(); // Should not throw + + Assertions.assertEquals("other", restProps.getTokenProvider()); + } + + @Test + public void testWarehouseProperty() { + Map props = new HashMap<>(); + props.put("paimon.rest.uri", "http://localhost:8080"); + props.put("warehouse", "s3://my-warehouse/path"); + props.put("paimon.rest.token.provider", "none"); + + PaimonRestMetaStoreProperties restProps = new PaimonRestMetaStoreProperties(props); + restProps.initNormalizeAndCheckProps(); + + // Warehouse property should be accessible through parent class + Assertions.assertNotNull(restProps); + } + + @Test + public void testCaseInsensitiveDlfTokenProvider() { + // Test that "DLF" is also recognized (case insensitive in validation) + Map props = new HashMap<>(); + props.put("paimon.rest.uri", "http://localhost:8080"); + props.put("paimon.rest.token.provider", "DLF"); + props.put("paimon.rest.dlf.access-key-id", "AKIAIOSFODNN7EXAMPLE"); + props.put("paimon.rest.dlf.access-key-secret", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); + props.put("warehouse", "catalog_name"); + + PaimonRestMetaStoreProperties restProps = new PaimonRestMetaStoreProperties(props); + restProps.initNormalizeAndCheckProps(); + + Assertions.assertEquals("DLF", restProps.getTokenProvider()); + } + + @Test + public void testMixedCaseTokenProvider() { + // Test mixed case token provider + Map props = new HashMap<>(); + props.put("paimon.rest.uri", "http://localhost:8080"); + props.put("paimon.rest.token.provider", "DlF"); + props.put("paimon.rest.dlf.access-key-id", "ak"); + props.put("paimon.rest.dlf.access-key-secret", "sk"); + props.put("warehouse", "catalog_name"); + + PaimonRestMetaStoreProperties restProps = new PaimonRestMetaStoreProperties(props); + restProps.initNormalizeAndCheckProps(); + + Assertions.assertEquals("DlF", restProps.getTokenProvider()); + } + + @Test + public void testTokenProviderValidationLogic() { + // Test: Token provider is required - should throw when missing + Map props1 = new HashMap<>(); + props1.put("paimon.rest.uri", "http://localhost:8080"); + + PaimonRestMetaStoreProperties restProps1 = new PaimonRestMetaStoreProperties(props1); + IllegalArgumentException exception1 = Assertions.assertThrows( + IllegalArgumentException.class, restProps1::initNormalizeAndCheckProps); + Assertions.assertTrue(exception1.getMessage().contains("paimon.rest.token.provider")); + + // Test: Token provider is required - should throw when empty + Map props2 = new HashMap<>(); + props2.put("paimon.rest.uri", "http://localhost:8080"); + props2.put("paimon.rest.token.provider", ""); + + PaimonRestMetaStoreProperties restProps2 = new PaimonRestMetaStoreProperties(props2); + IllegalArgumentException exception2 = Assertions.assertThrows( + IllegalArgumentException.class, restProps2::initNormalizeAndCheckProps); + Assertions.assertTrue(exception2.getMessage().contains("paimon.rest.token.provider")); + + // Test: Valid non-dlf token provider should work + Map props3 = new HashMap<>(); + props3.put("paimon.rest.uri", "http://localhost:8080"); + props3.put("paimon.rest.token.provider", "oauth2"); + props3.put("warehouse", "catalog_name"); + + PaimonRestMetaStoreProperties restProps3 = new PaimonRestMetaStoreProperties(props3); + restProps3.initNormalizeAndCheckProps(); // Should not throw + Assertions.assertEquals("oauth2", restProps3.getTokenProvider()); + } + + @Test + public void testDlfTokenProviderPositiveValidation() { + // Test: DLF token provider with all required credentials should work + Map props = new HashMap<>(); + props.put("paimon.rest.uri", "http://localhost:8080"); + props.put("paimon.rest.token.provider", "dlf"); + props.put("paimon.rest.dlf.access-key-id", "ak"); + props.put("paimon.rest.dlf.access-key-secret", "sk"); + props.put("warehouse", "catalog_name"); + + PaimonRestMetaStoreProperties restProps = new PaimonRestMetaStoreProperties(props); + restProps.initNormalizeAndCheckProps(); // Should not throw + + Assertions.assertEquals("dlf", restProps.getTokenProvider()); + } + + @Test + public void testDlfTokenProviderNegativeValidation() { + // Test: DLF token provider missing access-key-id should throw + Map props1 = new HashMap<>(); + props1.put("paimon.rest.uri", "http://localhost:8080"); + props1.put("paimon.rest.token.provider", "dlf"); + props1.put("warehouse", "catalog_name"); + props1.put("paimon.rest.dlf.access-key-secret", "sk"); + // Missing access-key-id + + PaimonRestMetaStoreProperties restProps1 = new PaimonRestMetaStoreProperties(props1); + IllegalArgumentException exception1 = Assertions.assertThrows( + IllegalArgumentException.class, restProps1::initNormalizeAndCheckProps); + String errorMessage1 = exception1.getMessage(); + Assertions.assertTrue(errorMessage1.contains("DLF token provider requires")); + Assertions.assertTrue(errorMessage1.contains("access-key-id")); + + // Test: DLF token provider missing secret-access-key should throw + Map props2 = new HashMap<>(); + props2.put("paimon.rest.uri", "http://localhost:8080"); + props2.put("paimon.rest.token.provider", "dlf"); + props2.put("warehouse", "catalog_name"); + props2.put("paimon.rest.dlf.access-key-id", "AKIAIOSFODNN7EXAMPLE"); + // Missing secret-access-key + + PaimonRestMetaStoreProperties restProps2 = new PaimonRestMetaStoreProperties(props2); + IllegalArgumentException exception2 = Assertions.assertThrows( + IllegalArgumentException.class, restProps2::initNormalizeAndCheckProps); + String errorMessage2 = exception2.getMessage(); + Assertions.assertTrue(errorMessage2.contains("DLF token provider requires")); + Assertions.assertTrue(errorMessage2.contains("access-key-secret")); + + // Test: DLF token provider missing both credentials should throw + Map props3 = new HashMap<>(); + props3.put("paimon.rest.uri", "http://localhost:8080"); + props3.put("paimon.rest.token.provider", "dlf"); + props3.put("warehouse", "catalog_name"); + // Missing both credentials + + PaimonRestMetaStoreProperties restProps3 = new PaimonRestMetaStoreProperties(props3); + IllegalArgumentException exception3 = Assertions.assertThrows( + IllegalArgumentException.class, restProps3::initNormalizeAndCheckProps); + String errorMessage3 = exception3.getMessage(); + Assertions.assertTrue(errorMessage3.contains("DLF token provider requires")); + } + + @Test + public void testPaimonRestPropertiesWithMultipleCustomProperties() { + Map props = new HashMap<>(); + props.put("paimon.rest.uri", "http://localhost:8080"); + props.put("paimon.rest.custom.auth.token", "token123"); + props.put("paimon.rest.custom.header.x-api-key", "api-key-456"); + props.put("paimon.rest.custom.ssl.verify", "false"); + props.put("non.paimon.property", "should-not-be-included"); + props.put("paimon.rest.token.provider", "none"); + props.put("warehouse", "catalog_name"); + + PaimonRestMetaStoreProperties restProps = new PaimonRestMetaStoreProperties(props); + restProps.initNormalizeAndCheckProps(); + + List storagePropertiesList = new ArrayList<>(); + restProps.buildCatalogOptions(storagePropertiesList); + Options catalogOptions = restProps.getCatalogOptions(); + + // paimon.rest.* properties should be passed through without prefix + Assertions.assertEquals("token123", catalogOptions.get("custom.auth.token")); + Assertions.assertEquals("api-key-456", catalogOptions.get("custom.header.x-api-key")); + Assertions.assertEquals("false", catalogOptions.get("custom.ssl.verify")); + + // Non-paimon.rest properties should not be included + Assertions.assertNull(catalogOptions.get("non.paimon.property")); + Assertions.assertNull(catalogOptions.get("should-not-be-included")); + } + + @Test + public void testMissingTokenProviderThrowsException() { + Map props = new HashMap<>(); + props.put("paimon.rest.uri", "http://localhost:8080"); + // Missing token provider + + PaimonRestMetaStoreProperties restProps = new PaimonRestMetaStoreProperties(props); + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, restProps::initNormalizeAndCheckProps); + + String errorMessage = exception.getMessage(); + Assertions.assertTrue(errorMessage.contains("paimon.rest.token.provider")); + } + + @Test + public void testEmptyTokenProviderThrowsException() { + Map props = new HashMap<>(); + props.put("paimon.rest.uri", "http://localhost:8080"); + props.put("paimon.rest.token.provider", ""); + + PaimonRestMetaStoreProperties restProps = new PaimonRestMetaStoreProperties(props); + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, restProps::initNormalizeAndCheckProps); + + String errorMessage = exception.getMessage(); + Assertions.assertTrue(errorMessage.contains("paimon.rest.token.provider")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/MinioPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/MinioPropertiesTest.java index d4418d2f36d832..fe6d8c5e859a21 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/MinioPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/MinioPropertiesTest.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.property.storage; +import org.apache.doris.common.ExceptionChecker; import org.apache.doris.common.UserException; import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException; @@ -50,12 +51,10 @@ public void testValidMinioConfiguration() throws UserException { Assertions.assertEquals("us-east-1", minioProperties.getRegion()); origProps.remove("s3.endpoint"); origProps.put("uri", "http://localhost:9000/test/"); - Assertions.assertThrows(IllegalArgumentException.class, () -> - StorageProperties.createPrimary(origProps), "Property endpoint is required."); + ExceptionChecker.expectThrowsWithMsg(IllegalArgumentException.class, + "Property minio.endpoint is required", () -> StorageProperties.createPrimary(origProps)); origProps.put("s3.endpoint", "http://localhost:9000"); - Assertions.assertDoesNotThrow(() -> StorageProperties.createPrimary(origProps)); - - + ExceptionChecker.expectThrowsNoException(() -> StorageProperties.createPrimary(origProps)); } @Test @@ -68,18 +67,18 @@ public void testGuessIsMeWithMinio() { public void testMissingAccessKey() { origProps.put("s3.endpoint", "http://localhost:9000"); origProps.put("s3.secret_key", "minioSecretKey"); - Assertions.assertThrows(StoragePropertiesException.class, () -> - StorageProperties.createPrimary(origProps), - "Please set access_key and secret_key or omit both for anonymous access to public bucket."); + ExceptionChecker.expectThrowsWithMsg(StoragePropertiesException.class, + "Please set access_key and secret_key or omit both for anonymous access to public bucket.", + () -> StorageProperties.createPrimary(origProps)); } @Test public void testMissingSecretKey() { origProps.put("s3.endpoint", "http://localhost:9000"); origProps.put("s3.access_key", "minioAccessKey"); - Assertions.assertThrows(StoragePropertiesException.class, () -> - StorageProperties.createPrimary(origProps), - "Please set access_key and secret_key or omit both for anonymous access to public bucket."); + ExceptionChecker.expectThrowsWithMsg(StoragePropertiesException.class, + "Please set access_key and secret_key or omit both for anonymous access to public bucket.", + () -> StorageProperties.createPrimary(origProps)); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java index e6c7fe3d582c06..e45e5b07553eff 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.property.storage; +import org.apache.doris.common.ExceptionChecker; import org.apache.doris.common.UserException; import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException; @@ -36,18 +37,20 @@ public void testBasicCreateTest() throws UserException { origProps.put("obs.access_key", "myOBSAccessKey"); origProps.put("obs.secret_key", "myOBSSecretKey"); origProps.put(StorageProperties.FS_OBS_SUPPORT, "true"); - Assertions.assertThrows(IllegalArgumentException.class, () -> StorageProperties.createAll(origProps), "Invalid endpoint format: https://obs.example.com"); + ExceptionChecker.expectThrowsWithMsg(IllegalArgumentException.class, + "Invalid endpoint: https://obs.example.com", () -> StorageProperties.createAll(origProps)); // Test creation without additional properties origProps = new HashMap<>(); - origProps.put("obs.endpoint", "https://obs.example.com"); + origProps.put("obs.endpoint", "obs.cn-north-4.myhuaweicloud.com"); origProps.put(StorageProperties.FS_OBS_SUPPORT, "true"); - - Assertions.assertThrows(IllegalArgumentException.class, () -> StorageProperties.createAll(origProps), "Property obs.access_key is required."); + // allow both access_key and secret_key to be empty for anonymous access + ExceptionChecker.expectThrowsNoException(() -> StorageProperties.createAll(origProps)); origProps.put("obs.access_key", "myOBSAccessKey"); - Assertions.assertThrows(IllegalArgumentException.class, () -> StorageProperties.createAll(origProps), "Property obs.secret_key is required."); + ExceptionChecker.expectThrowsWithMsg(StoragePropertiesException.class, + "Please set access_key and secret_key or omit both for anonymous access to public bucket.", + () -> StorageProperties.createAll(origProps)); origProps.put("obs.secret_key", "myOBSSecretKey"); - Assertions.assertThrows(IllegalArgumentException.class, () -> StorageProperties.createAll(origProps), "Invalid endpoint format: https://obs.example.com"); origProps.put("obs.endpoint", "obs.cn-north-4.myhuaweicloud.com"); Assertions.assertDoesNotThrow(() -> StorageProperties.createAll(origProps)); origProps.put("obs.endpoint", "https://obs.cn-north-4.myhuaweicloud.com"); @@ -95,48 +98,51 @@ public void testToNativeS3Configuration() throws UserException { @Test public void testGetRegion() throws UserException { origProps.put("obs.endpoint", "obs.cn-north-4.myhuaweicloud.com"); - origProps.put("obs.access_key", "myCOSAccessKey"); - origProps.put("obs.secret_key", "myCOSSecretKey"); + origProps.put("obs.access_key", "myOBSAccessKey"); + origProps.put("obs.secret_key", "myOBSSecretKey"); OBSProperties obsProperties = (OBSProperties) StorageProperties.createAll(origProps).get(0); Assertions.assertEquals("cn-north-4", obsProperties.getRegion()); - Assertions.assertEquals("myCOSAccessKey", obsProperties.getAccessKey()); - Assertions.assertEquals("myCOSSecretKey", obsProperties.getSecretKey()); + Assertions.assertEquals("myOBSAccessKey", obsProperties.getAccessKey()); + Assertions.assertEquals("myOBSSecretKey", obsProperties.getSecretKey()); Assertions.assertEquals("obs.cn-north-4.myhuaweicloud.com", obsProperties.getEndpoint()); } @Test public void testGetRegionWithDefault() throws UserException { origProps.put("uri", "https://examplebucket-1250000000.obs.cn-north-4.myhuaweicloud.com/test/file.txt"); - origProps.put("obs.access_key", "myCOSAccessKey"); - origProps.put("obs.secret_key", "myCOSSecretKey"); + origProps.put("obs.access_key", "myOBSAccessKey"); + origProps.put("obs.secret_key", "myOBSSecretKey"); OBSProperties obsProperties = (OBSProperties) StorageProperties.createPrimary(origProps); Assertions.assertEquals("cn-north-4", obsProperties.getRegion()); - Assertions.assertEquals("myCOSAccessKey", obsProperties.getAccessKey()); - Assertions.assertEquals("myCOSSecretKey", obsProperties.getSecretKey()); + Assertions.assertEquals("myOBSAccessKey", obsProperties.getAccessKey()); + Assertions.assertEquals("myOBSSecretKey", obsProperties.getSecretKey()); Assertions.assertEquals("obs.cn-north-4.myhuaweicloud.com", obsProperties.getEndpoint()); - Map cosNoEndpointProps = new HashMap<>(); - cosNoEndpointProps.put("obs.access_key", "myCOSAccessKey"); - cosNoEndpointProps.put("obs.secret_key", "myCOSSecretKey"); - cosNoEndpointProps.put("obs.region", "ap-beijing"); - cosNoEndpointProps.put("uri", "s3://examplebucket-1250000000/myhuaweicloud.com/test/file.txt"); + Map obsNoEndpointProps = new HashMap<>(); + obsNoEndpointProps.put("obs.access_key", "myOBSAccessKey"); + obsNoEndpointProps.put("obs.secret_key", "myOBSSecretKey"); + obsNoEndpointProps.put("obs.region", "ap-beijing"); + obsNoEndpointProps.put("uri", "s3://examplebucket-1250000000/myhuaweicloud.com/test/file.txt"); //not support - Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> StorageProperties.createPrimary(cosNoEndpointProps), "Property cos.endpoint is required."); + ExceptionChecker.expectThrowsWithMsg(IllegalArgumentException.class, + "Property obs.endpoint is required.", () -> StorageProperties.createPrimary(obsNoEndpointProps)); } @Test public void testmissingAccessKey() { origProps.put("obs.endpoint", "obs.cn-north-4.myhuaweicloud.com"); - origProps.put("obs.secret_key", "myCOSSecretKey"); - Assertions.assertThrows(StoragePropertiesException.class, () -> StorageProperties.createPrimary(origProps), - "Please set access_key and secret_key or omit both for anonymous access to public bucket."); + origProps.put("obs.secret_key", "myOBSSecretKey"); + ExceptionChecker.expectThrowsWithMsg(StoragePropertiesException.class, + "Please set access_key and secret_key or omit both for anonymous access to public bucket.", + () -> StorageProperties.createPrimary(origProps)); } @Test public void testMissingSecretKey() { origProps.put("obs.endpoint", "obs.cn-north-4.myhuaweicloud.com"); - origProps.put("obs.access_key", "myCOSAccessKey"); - Assertions.assertThrows(StoragePropertiesException.class, () -> StorageProperties.createPrimary(origProps), - "Please set access_key and secret_key or omit both for anonymous access to public bucket."); + origProps.put("obs.access_key", "myOBSAccessKey"); + ExceptionChecker.expectThrowsWithMsg(StoragePropertiesException.class, + "Please set access_key and secret_key or omit both for anonymous access to public bucket.", + () -> StorageProperties.createPrimary(origProps)); } private static String obsAccessKey = ""; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java index ede08f399fc637..0eee9b497615c6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.property.storage; +import org.apache.doris.common.ExceptionChecker; import org.apache.doris.common.UserException; import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException; @@ -40,7 +41,8 @@ public void testBasicCreateTest() { origProps.put("oss.secret_key", "myOSSSecretKey"); origProps.put(StorageProperties.FS_OSS_SUPPORT, "true"); Map finalOrigProps = origProps; - Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> StorageProperties.createPrimary(finalOrigProps), "Property oss.endpoint is required."); + ExceptionChecker.expectThrowsWithMsg(IllegalArgumentException.class, + "Invalid endpoint: https://oss.aliyuncs.com", () -> StorageProperties.createPrimary(finalOrigProps)); origProps.put("oss.endpoint", "oss-cn-shenzhen-finance-1-internal.aliyuncs.com"); Map finalOrigProps1 = origProps; OSSProperties ossProperties = (OSSProperties) StorageProperties.createPrimary(finalOrigProps1); @@ -48,12 +50,12 @@ public void testBasicCreateTest() { Assertions.assertEquals("cn-shenzhen-finance-1", ossProperties.getRegion()); Assertions.assertDoesNotThrow(() -> StorageProperties.createPrimary(finalOrigProps1)); origProps = new HashMap<>(); - origProps.put("oss.endpoint", "https://oss.aliyuncs.com"); + origProps.put("oss.endpoint", "oss-cn-shenzhen-finance-1-internal.aliyuncs.com"); Map finalOrigProps2 = origProps; - Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> StorageProperties.createPrimary(finalOrigProps2)); + // allow both access_key and secret_key to be empty for anonymous access + ExceptionChecker.expectThrowsNoException(() -> StorageProperties.createPrimary(finalOrigProps2)); } - @Test public void testToNativeS3Configuration() throws UserException { Map origProps = new HashMap<>(); @@ -98,12 +100,12 @@ public void testToNativeS3Configuration() throws UserException { public void testGetRegion() throws UserException { Map origProps = new HashMap<>(); origProps.put("oss.endpoint", "oss-cn-hangzhou.aliyuncs.com"); - origProps.put("oss.access_key", "myCOSAccessKey"); - origProps.put("oss.secret_key", "myCOSSecretKey"); + origProps.put("oss.access_key", "myOSSAccessKey"); + origProps.put("oss.secret_key", "myOSSSecretKey"); OSSProperties ossProperties = (OSSProperties) StorageProperties.createPrimary(origProps); Assertions.assertEquals("cn-hangzhou", ossProperties.getRegion()); - Assertions.assertEquals("myCOSAccessKey", ossProperties.getAccessKey()); - Assertions.assertEquals("myCOSSecretKey", ossProperties.getSecretKey()); + Assertions.assertEquals("myOSSAccessKey", ossProperties.getAccessKey()); + Assertions.assertEquals("myOSSSecretKey", ossProperties.getSecretKey()); Assertions.assertEquals("oss-cn-hangzhou.aliyuncs.com", ossProperties.getEndpoint()); origProps.put("oss.endpoint", "oss-cn-hangzhou-internal.aliyuncs.com"); Assertions.assertEquals("cn-hangzhou", ((OSSProperties) StorageProperties.createPrimary(origProps)).getRegion()); @@ -123,20 +125,20 @@ public void testGetRegion() throws UserException { public void testGetRegionWithDefault() throws UserException { Map origProps = new HashMap<>(); origProps.put("uri", "https://examplebucket-1250000000.oss-cn-hangzhou.aliyuncs.com/test/file.txt"); - origProps.put("oss.access_key", "myCOSAccessKey"); - origProps.put("oss.secret_key", "myCOSSecretKey"); + origProps.put("oss.access_key", "myOSSAccessKey"); + origProps.put("oss.secret_key", "myOSSSecretKey"); OSSProperties ossProperties = (OSSProperties) StorageProperties.createPrimary(origProps); Assertions.assertEquals("cn-hangzhou", ossProperties.getRegion()); - Assertions.assertEquals("myCOSAccessKey", ossProperties.getAccessKey()); - Assertions.assertEquals("myCOSSecretKey", ossProperties.getSecretKey()); + Assertions.assertEquals("myOSSAccessKey", ossProperties.getAccessKey()); + Assertions.assertEquals("myOSSSecretKey", ossProperties.getSecretKey()); Assertions.assertEquals("oss-cn-hangzhou.aliyuncs.com", ossProperties.getEndpoint()); - Map cosNoEndpointProps = new HashMap<>(); - cosNoEndpointProps.put("oss.access_key", "myCOSAccessKey"); - cosNoEndpointProps.put("oss.secret_key", "myCOSSecretKey"); - cosNoEndpointProps.put("oss.region", "cn-hangzhou"); + Map ossNoEndpointProps = new HashMap<>(); + ossNoEndpointProps.put("oss.access_key", "myOSSAccessKey"); + ossNoEndpointProps.put("oss.secret_key", "myOSSSecretKey"); + ossNoEndpointProps.put("oss.region", "cn-hangzhou"); origProps.put("uri", "s3://examplebucket-1250000000/test/file.txt"); - // not support - Assertions.assertThrowsExactly(StoragePropertiesException.class, () -> StorageProperties.createPrimary(cosNoEndpointProps), "Property cos.endpoint is required."); + // oss support without endpoint + ExceptionChecker.expectThrowsNoException(() -> StorageProperties.createPrimary(ossNoEndpointProps)); } @Test @@ -144,8 +146,9 @@ public void testMissingAccessKey() { Map origProps = new HashMap<>(); origProps.put("oss.endpoint", "oss-cn-hangzhou.aliyuncs.com"); origProps.put("oss.secret_key", "myOSSSecretKey"); - Assertions.assertThrows(StoragePropertiesException.class, () -> StorageProperties.createPrimary(origProps), - "Please set access_key and secret_key or omit both for anonymous access to public bucket."); + ExceptionChecker.expectThrowsWithMsg(StoragePropertiesException.class, + "Please set access_key and secret_key or omit both for anonymous access to public bucket.", + () -> StorageProperties.createPrimary(origProps)); } @Test @@ -153,8 +156,9 @@ public void testMissingSecretKey() { Map origProps = new HashMap<>(); origProps.put("oss.endpoint", "oss-cn-hangzhou.aliyuncs.com"); origProps.put("oss.access_key", "myOSSAccessKey"); - Assertions.assertThrows(StoragePropertiesException.class, () -> StorageProperties.createPrimary(origProps), - "Please set access_key and secret_key or omit both for anonymous access to public bucket."); + ExceptionChecker.expectThrowsWithMsg(StoragePropertiesException.class, + "Please set access_key and secret_key or omit both for anonymous access to public bucket.", + () -> StorageProperties.createPrimary(origProps)); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java index 23e3c99e309ac9..a766be98f15282 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.property.storage; +import org.apache.doris.common.ExceptionChecker; import org.apache.doris.common.UserException; import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException; @@ -58,19 +59,22 @@ public void testS3Properties() { origProps.put("s3.secret_key", "myS3SecretKey"); origProps.put("s3.region", "us-west-1"); origProps.put(StorageProperties.FS_S3_SUPPORT, "true"); + ExceptionChecker.expectThrowsWithMsg(IllegalArgumentException.class, + "Invalid endpoint: https://cos.example.com", () -> StorageProperties.createAll(origProps)); origProps = new HashMap<>(); - origProps.put("s3.endpoint", "https://s3.example.com"); + origProps.put("s3.endpoint", "s3-fips.dualstack.us-east-2.amazonaws.com"); origProps.put(StorageProperties.FS_S3_SUPPORT, "true"); - Assertions.assertThrows(IllegalArgumentException.class, () -> StorageProperties.createAll(origProps), - "Property cos.access_key is required."); + S3Properties s3Properties = (S3Properties) StorageProperties.createPrimary(origProps); + Assertions.assertEquals("us-east-2", s3Properties.getRegion()); + Assertions.assertEquals("s3-fips.dualstack.us-east-2.amazonaws.com", s3Properties.getEndpoint()); + + origProps = new HashMap<>(); + origProps.put("s3.endpoint", "s3-fips.dualstack.us-east-2.amazonaws.com"); origProps.put("s3.access_key", "myS3AccessKey"); - Assertions.assertThrows(IllegalArgumentException.class, () -> StorageProperties.createAll(origProps), - "Property cos.secret_key is required."); + ExceptionChecker.expectThrowsWithMsg(StoragePropertiesException.class, + "Please set s3.access_key and s3.secret_key", () -> StorageProperties.createAll(origProps)); origProps.put("s3.secret_key", "myS3SecretKey"); - Assertions.assertThrows(IllegalArgumentException.class, () -> StorageProperties.createAll(origProps), - "Invalid endpoint format: https://s3.example.com"); - origProps.put("s3.endpoint", "s3.us-west-1.amazonaws.com"); - Assertions.assertDoesNotThrow(() -> StorageProperties.createAll(origProps)); + ExceptionChecker.expectThrowsNoException(() -> StorageProperties.createAll(origProps)); } @Test @@ -126,10 +130,10 @@ public void testToNativeS3Configuration() throws UserException { origProps.put("s3.connection.timeout", "6000"); origProps.put("test_non_storage_param", "6000"); - - Assertions.assertThrows(IllegalArgumentException.class, () -> { - StorageProperties.createAll(origProps).get(1); - }, "Invalid endpoint format: https://cos.example.com"); + ExceptionChecker.expectThrowsWithMsg(IllegalArgumentException.class, + "Invalid endpoint: https://cos.example.com", () -> { + StorageProperties.createAll(origProps).get(1); + }); origProps.put("s3.endpoint", "s3.us-west-1.amazonaws.com"); S3Properties s3Properties = (S3Properties) StorageProperties.createAll(origProps).get(0); Map s3Props = s3Properties.getBackendConfigProperties(); @@ -203,8 +207,7 @@ public void testGetRegionWithDefault() throws UserException { s3EndpointProps.put("oss.region", "cn-hangzhou"); origProps.put("uri", "s3://examplebucket-1250000000/test/file.txt"); // not support - Assertions.assertThrowsExactly(StoragePropertiesException.class, - () -> StorageProperties.createPrimary(s3EndpointProps), "Property cos.endpoint is required."); + ExceptionChecker.expectThrowsNoException(() -> StorageProperties.createPrimary(s3EndpointProps)); } @Test @@ -308,17 +311,22 @@ public void testInvalidEndpoint() { // Fails because it contains 'amazonaws.com' but doesn't match the strict S3 endpoint pattern (missing region). String invalidEndpoint1 = "s3.amazonaws.com"; origProps.put("s3.endpoint", invalidEndpoint1); - Assertions.assertThrows(IllegalArgumentException.class, () -> StorageProperties.createPrimary(origProps)); + ExceptionChecker.expectThrowsWithMsg(IllegalArgumentException.class, + "Invalid endpoint: s3.amazonaws.com", () -> StorageProperties.createPrimary(origProps)); // Fails because it contains 'amazonaws.com' but doesn't match the strict S3 endpoint pattern (invalid subdomain). String invalidEndpoint2 = "my-s3-service.amazonaws.com"; origProps.put("s3.endpoint", invalidEndpoint2); - Assertions.assertThrows(IllegalArgumentException.class, () -> StorageProperties.createPrimary(origProps)); + ExceptionChecker.expectThrowsWithMsg(IllegalArgumentException.class, + "Invalid endpoint: my-s3-service.amazonaws.com", + () -> StorageProperties.createPrimary(origProps)); // Fails because it contains 'amazonaws.com' but doesn't match the strict S3 endpoint pattern (invalid TLD). String invalidEndpoint3 = "http://s3.us-west-2.amazonaws.com.cn"; origProps.put("s3.endpoint", invalidEndpoint3); - Assertions.assertThrows(IllegalArgumentException.class, () -> StorageProperties.createPrimary(origProps)); + ExceptionChecker.expectThrowsWithMsg(IllegalArgumentException.class, + "Invalid endpoint: http://s3.us-west-2.amazonaws.com.cn", + () -> StorageProperties.createPrimary(origProps)); } @Test diff --git a/regression-test/data/external_table_p2/paimon/test_paimon_dlf_rest_catalog.out b/regression-test/data/external_table_p2/paimon/test_paimon_dlf_rest_catalog.out new file mode 100644 index 00000000000000..60087bd0be5f27 --- /dev/null +++ b/regression-test/data/external_table_p2/paimon/test_paimon_dlf_rest_catalog.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !c1 -- +1 25-34 M true +2 18-24 F false + +-- !c1 -- +1 25-34 M true +2 18-24 F false + diff --git a/regression-test/suites/external_table_p2/iceberg/test_glue_rest_s3tables.groovy b/regression-test/suites/external_table_p2/iceberg/test_glue_rest_s3tables.groovy index 1954ce0a231708..8ad9e0e28897ed 100644 --- a/regression-test/suites/external_table_p2/iceberg/test_glue_rest_s3tables.groovy +++ b/regression-test/suites/external_table_p2/iceberg/test_glue_rest_s3tables.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_glue_rest_s3tables", "p2,external,iceberg,external_remote,external_remote_iceberg") { +suite("test_glue_rest_s3tables", "p2,external,iceberg,external_remote,external_remote_iceberg,new_catalog_property") { def format_compressions = ["parquet_zstd"] def q01 = { String format_compression, String catalog_name -> @@ -651,9 +651,9 @@ suite("test_glue_rest_s3tables", "p2,external,iceberg,external_remote,external_r """ sql """ switch ${catalog_name};""" - sql """ drop database if exists iceberg_s3tables_glue_rest force""" - sql """ create database iceberg_s3tables_glue_rest""" - sql """ use iceberg_s3tables_glue_rest;""" + sql """ drop database if exists iceberg_s3tables_glue_rest_master force""" + sql """ create database iceberg_s3tables_glue_rest_master""" + sql """ use iceberg_s3tables_glue_rest_master;""" sql """ set enable_fallback_to_original_planner=false """ try { diff --git a/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_rest_catalog.groovy b/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_rest_catalog.groovy new file mode 100644 index 00000000000000..08946964b7c099 --- /dev/null +++ b/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_rest_catalog.groovy @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_dlf_rest_catalog", "p2,external,paimon,external_remote,external_remote_paimon,new_catalog_property") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + try { + String catalog = "test_paimon_dlf_rest_catalog" + String prop= context.config.otherConfigs.get("paimonDlfRestCatalog") + + sql """drop catalog if exists ${catalog};""" + sql """ + create catalog if not exists ${catalog} properties ( + ${prop} + ); + """ + + sql """ use ${catalog}.new_dlf_paimon_db""" + + sql """set force_jni_scanner=false""" + qt_c1 """ select * from users_samples order by user_id """ + sql """select * from users_samples\$files;""" + + sql """set force_jni_scanner=true""" + qt_c1 """ select * from users_samples order by user_id """ + sql """select * from users_samples\$files;""" + + } finally { + sql """set force_jni_scanner=false""" + } +} + diff --git a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy index 8f8ffac9c98e35..927c267718ba94 100644 --- a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy +++ b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy @@ -102,7 +102,7 @@ suite("test_domain_connection_and_ak_sk_correction", "load_p0") { assertTrue(false. "The endpoint is wrong, so the connection test should fale") } catch (Exception e) { logger.info("the second sql exception result is {}", e.getMessage()) - assertTrue(e.getMessage().contains("Invalid endpoint format"), e.getMessage()) + assertTrue(e.getMessage().contains("Invalid endpoint"), e.getMessage()) } sql """ DROP TABLE IF EXISTS ${tableName} FORCE"""