diff --git a/build.sh b/build.sh
index 55c5bacb919001..1d1c69ed5fd5eb 100755
--- a/build.sh
+++ b/build.sh
@@ -795,6 +795,10 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then
modules+=("fe-extension-spi")
modules+=("fe-extension-loader")
modules+=("fe-core")
+ # Hadoop as one shared library bundle for the plugins, unpacked into plugins/shared/hadoop/
+ # below. Nothing depends on it, so -am does not reach it; it has to be named here or the zip is
+ # simply never built and the deploy step below silently ships whatever an older build left.
+ modules+=("fe-hadoop-runtime")
# Filesystem API and SPI plugin modules (loaded at runtime as plugins)
modules+=("fe-filesystem/fe-filesystem-api")
modules+=("fe-filesystem/fe-filesystem-spi")
@@ -1269,6 +1273,30 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then
mkdir -p "${DORIS_OUTPUT}/fe/plugins/hadoop_conf/"
mkdir -p "${DORIS_OUTPUT}/fe/plugins/java_extensions/"
+ # Shared library bundles: libraries that every plugin needing them must load from ONE place,
+ # because their classes inherit across jars and their static state (FileSystem.CACHE, the UGI
+ # login, the FileSystem service registry) is what makes two plugins agree about one cluster.
+ # SharedLibraryLayer turns this directory into the parent classloader of every filesystem and
+ # connector plugin. Deliberately at plugins/ level and not under plugins/filesystem or
+ # plugins/connector: DirectoryPluginRuntimeManager treats every direct subdirectory of those as
+ # a plugin, and would report this one as a plugin that failed to load.
+ # NOTE: plugins/shared/ is NOT added to the FE CLASSPATH - it is loaded by FE, not by the JVM
+ # launcher. See bin/start_fe.sh.
+ # Created only when there is a bundle to put in it - the mkdir below already builds the whole
+ # chain. An empty plugins/shared would occupy that name in Config.plugin_dir for nothing.
+ SHARED_LIB_DIR="${DORIS_OUTPUT}/fe/plugins/shared"
+ HADOOP_RUNTIME_ZIP="${DORIS_HOME}/fe/fe-hadoop-runtime/target/doris-fe-hadoop-runtime.zip"
+ if [[ -f "${HADOOP_RUNTIME_ZIP}" ]]; then
+ # Same rule as the plugin directories: unzip -o overwrites but never removes, so a version
+ # bump would leave both copies of every versioned jar here and the layer would bind whichever
+ # the sorted URL order reached first. Clear what the zip owns and unpack fresh.
+ rm -rf "${SHARED_LIB_DIR}/hadoop/lib"
+ rm -f "${SHARED_LIB_DIR}/hadoop"/*.jar
+ mkdir -p "${SHARED_LIB_DIR}/hadoop"
+ unzip -q -o "${HADOOP_RUNTIME_ZIP}" -d "${SHARED_LIB_DIR}/hadoop/"
+ fi
+ unset SHARED_LIB_DIR HADOOP_RUNTIME_ZIP
+
# Deploy filesystem provider plugins as independent plugin directories
# Each sub-directory is one storage backend loaded at runtime by FileSystemPluginManager.
FS_PLUGIN_DIR="${DORIS_OUTPUT}/fe/plugins/filesystem"
diff --git a/fe/be-java-extensions/java-udf/pom.xml b/fe/be-java-extensions/java-udf/pom.xml
index 60a98854ba2ad3..6d323f350e0d19 100644
--- a/fe/be-java-extensions/java-udf/pom.xml
+++ b/fe/be-java-extensions/java-udf/pom.xml
@@ -68,12 +68,16 @@ under the License.
dependency that is only reached transitively is one refactor away from disappearing.
The exclusions are what fe-common carries for the FE that no Java function executor can
- reach: netty, the S3 filesystem, protobuf, jackson, RoaringBitmap. They were found by
- walking jdeps' class-level edges out of this module and taking the jars with nothing
- reachable in them, then dropping the ones that are reached by ServiceLoader instead
- (the logging providers) or that exist to be extended rather than called
+ reach: netty, protobuf, jackson, RoaringBitmap, commons-io, commons-codec. They were
+ found by walking jdeps' class-level edges out of this module and taking the jars with
+ nothing reachable in them, then dropping the ones that are reached by ServiceLoader
+ instead (the logging providers) or that exist to be extended rather than called
(hive-udf-shade). Verified the only way it can be verified, by loading the plugin out
- of a real deployment directory with them gone.
+ of a real deployment directory with them gone. commons-codec is the one that used to be
+ excluded for free: fe-common reached it only through hadoop-common, at provided scope,
+ so it never arrived here; now that fe-common declares it for its own Hll, this list has
+ to say so or the jar walks in. The S3 filesystem (hadoop-aws) went the other way - it
+ left fe-common altogether - so it no longer needs naming here.
-->
org.apache.doris
@@ -84,10 +88,6 @@ under the License.
io.netty
netty-all
-
- org.apache.hadoop
- hadoop-aws
-
com.google.protobuf
protobuf-java
@@ -108,6 +108,10 @@ under the License.
commons-io
commons-io
+
+ commons-codec
+ commons-codec
+
org.apache.logging.log4j
log4j-iostreams
diff --git a/fe/fe-common/pom.xml b/fe/fe-common/pom.xml
index aa6e4631666ad7..83550350dae8e4 100644
--- a/fe/fe-common/pom.xml
+++ b/fe/fe-common/pom.xml
@@ -104,20 +104,12 @@ under the License.
org.projectlombok
lombok
+
- org.apache.hadoop
- hadoop-common
-
-
- commons-collections
- commons-collections
-
-
- org.apache.commons
- commons-compress
-
-
- provided
+ commons-codec
+ commons-codec
${project.groupId}
fe-common
@@ -709,7 +715,10 @@ under the License.
com.google.flatbuffers
flatbuffers-java
-
+ broke CREATE FUNCTION for every hive-derived UDF at runtime.
+
+ (2) It is also the provider of org.apache.hadoop.hive.ql.* for four connector plugins.
+ Three of them (paimon, hive, iceberg) bundle no hive-exec class at all; the fourth,
+ hudi, bundles one subset: fe-connector-hudi unpacks this artifact's ql/io/parquet/**
+ (156 classes) into its own jar, for the mapred/mapreduce hierarchy its pom explains.
+ Every other hive-exec reference they carry resolves to this jar through their
+ classloader - parent-first today via
+ ConnectorPluginManager.CONNECTOR_PARENT_FIRST_PREFIXES, which also makes the kernel's
+ copy of that hudi subset the one that loads; plain child-first fallback if that prefix
+ ever goes away, since dropping it changes the delegation order but not the
+ reachability. Measured over the built plugin zips (referenced minus self-carried, cut
+ against this artifact's class list): paimon 15 - paimon-hive's PaimonStorageHandler
+ implements ql.metadata.HiveStorageHandler, HiveCatalog.createView calls
+ ql.metadata.Table.getEmptyTable, HiveUtils uses ql.io.sarg.ConvertAstToSearchArg; hudi
+ 46 - hudi-hadoop-mr reaches 25 (HoodieCombineHiveInputFormat's ql.io.HiveInputFormat,
+ CombineHiveRecordReader and IOContextMap, the ORC branch of HoodieInputFormatUtils,
+ the ql.plan descriptors), and the unpacked parquet subset itself reaches 33
+ (ql.exec.Utilities, ql.io.HiveFileFormatUtils, ql.exec.vector.*,
+ llap.LlapCacheAwareFs, the same descriptors), so carrying that subset made the plugin
+ depend on MORE of this jar, not less; hive and iceberg 3 each - the bundled Aliyun DLF
+ metastore client calls ql.session.SessionState.get(). Swapping this for the 10-class
+ be-java-extensions/hive-udf-shade compiles green and breaks those paths at runtime, the
+ same shape of failure as the hive-catalog-shade removal in (1). Making the plugins
+ self-sufficient is the prerequisite for shrinking this, not a follow-up to it. -->
org.apache.hive
hive-exec
@@ -743,6 +776,20 @@ under the License.
hadoop-aws
+
+
+ io.dropwizard.metrics
+ metrics-core
+
+
+
+ 4.0.0
+
+ org.apache.doris
+ fe
+ ${revision}
+
+
+ fe-hadoop-runtime
+ pom
+ Doris FE Hadoop Runtime
+
+ Ships Hadoop as one shared library bundle for the FE's plugins, rather than a copy inside
+ each plugin that needs it. Produces doris-fe-hadoop-runtime.zip, unpacked by build.sh into
+ fe/plugins/shared/hadoop/ and turned into a classloader by SharedLibraryLayer.
+
+ This module has no sources. It exists to give the bundle one dependency resolution and one
+ artifact list, which is what makes "the same Hadoop for every plugin" checkable rather than
+ a coincidence of nine plugin poms agreeing.
+
+
+
+
+
+ ${project.groupId}
+ hadoop-deps
+ ${project.version}
+
+
+ *
+ *
+
+
+
+
+
+
+ org.apache.hadoop
+ hadoop-client
+
+
+ org.slf4j
+ slf4j-log4j12
+
+
+ log4j
+ log4j
+
+
+
+
+ org.apache.hadoop
+ hadoop-auth
+
+
+
+ org.apache.hadoop
+ hadoop-common
+
+
+
+
+ doris-fe-hadoop-runtime
+
+
+ maven-assembly-plugin
+
+ false
+
+ src/main/assembly/hadoop-runtime-zip.xml
+
+
+
+
+ make-assembly
+ package
+
+ single
+
+
+
+
+
+ maven-antrun-plugin
+ 3.1.0
+
+
+ verify-bundle-shape
+ package
+
+ run
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/fe/fe-hadoop-runtime/src/main/assembly/hadoop-runtime-zip.xml b/fe/fe-hadoop-runtime/src/main/assembly/hadoop-runtime-zip.xml
new file mode 100644
index 00000000000000..e7a4050648f3fb
--- /dev/null
+++ b/fe/fe-hadoop-runtime/src/main/assembly/hadoop-runtime-zip.xml
@@ -0,0 +1,122 @@
+
+
+
+
+ hadoop-runtime
+
+ zip
+
+ false
+
+
+
+
+ /
+ false
+ runtime
+ ${artifact.artifactId}.${artifact.extension}
+
+ org.apache.doris:hadoop-deps
+
+
+
+
+ /lib
+ false
+ runtime
+
+
+ org.apache.hadoop:hadoop-annotations
+ org.apache.hadoop:hadoop-auth
+ org.apache.hadoop:hadoop-client
+ org.apache.hadoop:hadoop-common
+ org.apache.hadoop:hadoop-hdfs-client
+ org.apache.hadoop:hadoop-mapreduce-client-common
+ org.apache.hadoop:hadoop-mapreduce-client-core
+ org.apache.hadoop:hadoop-mapreduce-client-jobclient
+ org.apache.hadoop:hadoop-yarn-client
+
+ org.apache.hadoop.thirdparty:hadoop-shaded-guava
+ org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_25
+
+ org.apache.kerby:kerb-core
+ org.apache.kerby:kerb-crypto
+ org.apache.kerby:kerb-util
+ org.apache.kerby:kerby-asn1
+ org.apache.kerby:kerby-config
+ org.apache.kerby:kerby-pkix
+ org.apache.kerby:kerby-util
+
+ com.fasterxml.woodstox:woodstox-core
+ org.codehaus.woodstox:stax2-api
+ org.apache.commons:commons-configuration2
+ commons-net:commons-net
+ dnsjava:dnsjava
+ com.jcraft:jsch
+ com.google.re2j:re2j
+
+ org.apache.curator:curator-client
+ org.apache.curator:curator-framework
+ org.apache.curator:curator-recipes
+
+
+
+
diff --git a/fe/fe-kerberos/src/main/java/org/apache/doris/kerberos/AuthenticationConfig.java b/fe/fe-kerberos/src/main/java/org/apache/doris/kerberos/AuthenticationConfig.java
index efd23b83b191a5..828ca21d91db02 100644
--- a/fe/fe-kerberos/src/main/java/org/apache/doris/kerberos/AuthenticationConfig.java
+++ b/fe/fe-kerberos/src/main/java/org/apache/doris/kerberos/AuthenticationConfig.java
@@ -17,6 +17,8 @@
package org.apache.doris.kerberos;
+import org.apache.doris.foundation.security.HadoopAuthConfigKeys;
+
import com.google.common.base.Strings;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -27,11 +29,14 @@
public abstract class AuthenticationConfig {
private static final Logger LOG = LogManager.getLogger(AuthenticationConfig.class);
- public static String HADOOP_USER_NAME = "hadoop.username";
- public static String HADOOP_KERBEROS_PRINCIPAL = "hadoop.kerberos.principal";
- public static String HADOOP_KERBEROS_KEYTAB = "hadoop.kerberos.keytab";
- public static String HADOOP_SECURITY_AUTH_TO_LOCAL = "hadoop.security.auth_to_local";
- public static String DORIS_KRB5_DEBUG = "doris.krb5.debug";
+ // The names themselves live in fe-foundation, so that a module naming one of these properties
+ // does not have to link against Hadoop just to spell it. Kept here as well because they are
+ // part of this class's published surface.
+ public static final String HADOOP_USER_NAME = HadoopAuthConfigKeys.HADOOP_USER_NAME;
+ public static final String HADOOP_KERBEROS_PRINCIPAL = HadoopAuthConfigKeys.HADOOP_KERBEROS_PRINCIPAL;
+ public static final String HADOOP_KERBEROS_KEYTAB = HadoopAuthConfigKeys.HADOOP_KERBEROS_KEYTAB;
+ public static final String HADOOP_SECURITY_AUTH_TO_LOCAL = HadoopAuthConfigKeys.HADOOP_SECURITY_AUTH_TO_LOCAL;
+ public static final String DORIS_KRB5_DEBUG = HadoopAuthConfigKeys.DORIS_KRB5_DEBUG;
private static final String DEFAULT_HADOOP_USERNAME = "hadoop";
/**
diff --git a/fe/pom.xml b/fe/pom.xml
index a04b3ad9838436..5ca799a3266233 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -235,6 +235,7 @@ under the License.
fe-extension-spi
fe-connector
fe-extension-loader
+ fe-hadoop-runtime
fe-sql-parser
fe-core
hive-udf