diff --git a/build.sh b/build.sh index 55c5bacb919001..1d1c69ed5fd5eb 100755 --- a/build.sh +++ b/build.sh @@ -795,6 +795,10 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then modules+=("fe-extension-spi") modules+=("fe-extension-loader") modules+=("fe-core") + # Hadoop as one shared library bundle for the plugins, unpacked into plugins/shared/hadoop/ + # below. Nothing depends on it, so -am does not reach it; it has to be named here or the zip is + # simply never built and the deploy step below silently ships whatever an older build left. + modules+=("fe-hadoop-runtime") # Filesystem API and SPI plugin modules (loaded at runtime as plugins) modules+=("fe-filesystem/fe-filesystem-api") modules+=("fe-filesystem/fe-filesystem-spi") @@ -1269,6 +1273,30 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then mkdir -p "${DORIS_OUTPUT}/fe/plugins/hadoop_conf/" mkdir -p "${DORIS_OUTPUT}/fe/plugins/java_extensions/" + # Shared library bundles: libraries that every plugin needing them must load from ONE place, + # because their classes inherit across jars and their static state (FileSystem.CACHE, the UGI + # login, the FileSystem service registry) is what makes two plugins agree about one cluster. + # SharedLibraryLayer turns this directory into the parent classloader of every filesystem and + # connector plugin. Deliberately at plugins/ level and not under plugins/filesystem or + # plugins/connector: DirectoryPluginRuntimeManager treats every direct subdirectory of those as + # a plugin, and would report this one as a plugin that failed to load. + # NOTE: plugins/shared/ is NOT added to the FE CLASSPATH - it is loaded by FE, not by the JVM + # launcher. See bin/start_fe.sh. + # Created only when there is a bundle to put in it - the mkdir below already builds the whole + # chain. An empty plugins/shared would occupy that name in Config.plugin_dir for nothing. + SHARED_LIB_DIR="${DORIS_OUTPUT}/fe/plugins/shared" + HADOOP_RUNTIME_ZIP="${DORIS_HOME}/fe/fe-hadoop-runtime/target/doris-fe-hadoop-runtime.zip" + if [[ -f "${HADOOP_RUNTIME_ZIP}" ]]; then + # Same rule as the plugin directories: unzip -o overwrites but never removes, so a version + # bump would leave both copies of every versioned jar here and the layer would bind whichever + # the sorted URL order reached first. Clear what the zip owns and unpack fresh. + rm -rf "${SHARED_LIB_DIR}/hadoop/lib" + rm -f "${SHARED_LIB_DIR}/hadoop"/*.jar + mkdir -p "${SHARED_LIB_DIR}/hadoop" + unzip -q -o "${HADOOP_RUNTIME_ZIP}" -d "${SHARED_LIB_DIR}/hadoop/" + fi + unset SHARED_LIB_DIR HADOOP_RUNTIME_ZIP + # Deploy filesystem provider plugins as independent plugin directories # Each sub-directory is one storage backend loaded at runtime by FileSystemPluginManager. FS_PLUGIN_DIR="${DORIS_OUTPUT}/fe/plugins/filesystem" diff --git a/fe/be-java-extensions/java-udf/pom.xml b/fe/be-java-extensions/java-udf/pom.xml index 60a98854ba2ad3..6d323f350e0d19 100644 --- a/fe/be-java-extensions/java-udf/pom.xml +++ b/fe/be-java-extensions/java-udf/pom.xml @@ -68,12 +68,16 @@ under the License. dependency that is only reached transitively is one refactor away from disappearing. The exclusions are what fe-common carries for the FE that no Java function executor can - reach: netty, the S3 filesystem, protobuf, jackson, RoaringBitmap. They were found by - walking jdeps' class-level edges out of this module and taking the jars with nothing - reachable in them, then dropping the ones that are reached by ServiceLoader instead - (the logging providers) or that exist to be extended rather than called + reach: netty, protobuf, jackson, RoaringBitmap, commons-io, commons-codec. They were + found by walking jdeps' class-level edges out of this module and taking the jars with + nothing reachable in them, then dropping the ones that are reached by ServiceLoader + instead (the logging providers) or that exist to be extended rather than called (hive-udf-shade). Verified the only way it can be verified, by loading the plugin out - of a real deployment directory with them gone. + of a real deployment directory with them gone. commons-codec is the one that used to be + excluded for free: fe-common reached it only through hadoop-common, at provided scope, + so it never arrived here; now that fe-common declares it for its own Hll, this list has + to say so or the jar walks in. The S3 filesystem (hadoop-aws) went the other way - it + left fe-common altogether - so it no longer needs naming here. --> org.apache.doris @@ -84,10 +88,6 @@ under the License. io.netty netty-all - - org.apache.hadoop - hadoop-aws - com.google.protobuf protobuf-java @@ -108,6 +108,10 @@ under the License. commons-io commons-io + + commons-codec + commons-codec + org.apache.logging.log4j log4j-iostreams diff --git a/fe/fe-common/pom.xml b/fe/fe-common/pom.xml index aa6e4631666ad7..83550350dae8e4 100644 --- a/fe/fe-common/pom.xml +++ b/fe/fe-common/pom.xml @@ -104,20 +104,12 @@ under the License. org.projectlombok lombok + - org.apache.hadoop - hadoop-common - - - commons-collections - commons-collections - - - org.apache.commons - commons-compress - - - provided + commons-codec + commons-codec ${project.groupId} fe-common @@ -709,7 +715,10 @@ under the License. com.google.flatbuffers flatbuffers-java - + broke CREATE FUNCTION for every hive-derived UDF at runtime. + + (2) It is also the provider of org.apache.hadoop.hive.ql.* for four connector plugins. + Three of them (paimon, hive, iceberg) bundle no hive-exec class at all; the fourth, + hudi, bundles one subset: fe-connector-hudi unpacks this artifact's ql/io/parquet/** + (156 classes) into its own jar, for the mapred/mapreduce hierarchy its pom explains. + Every other hive-exec reference they carry resolves to this jar through their + classloader - parent-first today via + ConnectorPluginManager.CONNECTOR_PARENT_FIRST_PREFIXES, which also makes the kernel's + copy of that hudi subset the one that loads; plain child-first fallback if that prefix + ever goes away, since dropping it changes the delegation order but not the + reachability. Measured over the built plugin zips (referenced minus self-carried, cut + against this artifact's class list): paimon 15 - paimon-hive's PaimonStorageHandler + implements ql.metadata.HiveStorageHandler, HiveCatalog.createView calls + ql.metadata.Table.getEmptyTable, HiveUtils uses ql.io.sarg.ConvertAstToSearchArg; hudi + 46 - hudi-hadoop-mr reaches 25 (HoodieCombineHiveInputFormat's ql.io.HiveInputFormat, + CombineHiveRecordReader and IOContextMap, the ORC branch of HoodieInputFormatUtils, + the ql.plan descriptors), and the unpacked parquet subset itself reaches 33 + (ql.exec.Utilities, ql.io.HiveFileFormatUtils, ql.exec.vector.*, + llap.LlapCacheAwareFs, the same descriptors), so carrying that subset made the plugin + depend on MORE of this jar, not less; hive and iceberg 3 each - the bundled Aliyun DLF + metastore client calls ql.session.SessionState.get(). Swapping this for the 10-class + be-java-extensions/hive-udf-shade compiles green and breaks those paths at runtime, the + same shape of failure as the hive-catalog-shade removal in (1). Making the plugins + self-sufficient is the prerequisite for shrinking this, not a follow-up to it. --> org.apache.hive hive-exec @@ -743,6 +776,20 @@ under the License. hadoop-aws + + + io.dropwizard.metrics + metrics-core + + + + 4.0.0 + + org.apache.doris + fe + ${revision} + + + fe-hadoop-runtime + pom + Doris FE Hadoop Runtime + + Ships Hadoop as one shared library bundle for the FE's plugins, rather than a copy inside + each plugin that needs it. Produces doris-fe-hadoop-runtime.zip, unpacked by build.sh into + fe/plugins/shared/hadoop/ and turned into a classloader by SharedLibraryLayer. + + This module has no sources. It exists to give the bundle one dependency resolution and one + artifact list, which is what makes "the same Hadoop for every plugin" checkable rather than + a coincidence of nine plugin poms agreeing. + + + + + + ${project.groupId} + hadoop-deps + ${project.version} + + + * + * + + + + + + + org.apache.hadoop + hadoop-client + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + + + org.apache.hadoop + hadoop-auth + + + + org.apache.hadoop + hadoop-common + + + + + doris-fe-hadoop-runtime + + + maven-assembly-plugin + + false + + src/main/assembly/hadoop-runtime-zip.xml + + + + + make-assembly + package + + single + + + + + + maven-antrun-plugin + 3.1.0 + + + verify-bundle-shape + package + + run + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/fe/fe-hadoop-runtime/src/main/assembly/hadoop-runtime-zip.xml b/fe/fe-hadoop-runtime/src/main/assembly/hadoop-runtime-zip.xml new file mode 100644 index 00000000000000..e7a4050648f3fb --- /dev/null +++ b/fe/fe-hadoop-runtime/src/main/assembly/hadoop-runtime-zip.xml @@ -0,0 +1,122 @@ + + + + + hadoop-runtime + + zip + + false + + + + + / + false + runtime + ${artifact.artifactId}.${artifact.extension} + + org.apache.doris:hadoop-deps + + + + + /lib + false + runtime + + + org.apache.hadoop:hadoop-annotations + org.apache.hadoop:hadoop-auth + org.apache.hadoop:hadoop-client + org.apache.hadoop:hadoop-common + org.apache.hadoop:hadoop-hdfs-client + org.apache.hadoop:hadoop-mapreduce-client-common + org.apache.hadoop:hadoop-mapreduce-client-core + org.apache.hadoop:hadoop-mapreduce-client-jobclient + org.apache.hadoop:hadoop-yarn-client + + org.apache.hadoop.thirdparty:hadoop-shaded-guava + org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_25 + + org.apache.kerby:kerb-core + org.apache.kerby:kerb-crypto + org.apache.kerby:kerb-util + org.apache.kerby:kerby-asn1 + org.apache.kerby:kerby-config + org.apache.kerby:kerby-pkix + org.apache.kerby:kerby-util + + com.fasterxml.woodstox:woodstox-core + org.codehaus.woodstox:stax2-api + org.apache.commons:commons-configuration2 + commons-net:commons-net + dnsjava:dnsjava + com.jcraft:jsch + com.google.re2j:re2j + + org.apache.curator:curator-client + org.apache.curator:curator-framework + org.apache.curator:curator-recipes + + + + diff --git a/fe/fe-kerberos/src/main/java/org/apache/doris/kerberos/AuthenticationConfig.java b/fe/fe-kerberos/src/main/java/org/apache/doris/kerberos/AuthenticationConfig.java index efd23b83b191a5..828ca21d91db02 100644 --- a/fe/fe-kerberos/src/main/java/org/apache/doris/kerberos/AuthenticationConfig.java +++ b/fe/fe-kerberos/src/main/java/org/apache/doris/kerberos/AuthenticationConfig.java @@ -17,6 +17,8 @@ package org.apache.doris.kerberos; +import org.apache.doris.foundation.security.HadoopAuthConfigKeys; + import com.google.common.base.Strings; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; @@ -27,11 +29,14 @@ public abstract class AuthenticationConfig { private static final Logger LOG = LogManager.getLogger(AuthenticationConfig.class); - public static String HADOOP_USER_NAME = "hadoop.username"; - public static String HADOOP_KERBEROS_PRINCIPAL = "hadoop.kerberos.principal"; - public static String HADOOP_KERBEROS_KEYTAB = "hadoop.kerberos.keytab"; - public static String HADOOP_SECURITY_AUTH_TO_LOCAL = "hadoop.security.auth_to_local"; - public static String DORIS_KRB5_DEBUG = "doris.krb5.debug"; + // The names themselves live in fe-foundation, so that a module naming one of these properties + // does not have to link against Hadoop just to spell it. Kept here as well because they are + // part of this class's published surface. + public static final String HADOOP_USER_NAME = HadoopAuthConfigKeys.HADOOP_USER_NAME; + public static final String HADOOP_KERBEROS_PRINCIPAL = HadoopAuthConfigKeys.HADOOP_KERBEROS_PRINCIPAL; + public static final String HADOOP_KERBEROS_KEYTAB = HadoopAuthConfigKeys.HADOOP_KERBEROS_KEYTAB; + public static final String HADOOP_SECURITY_AUTH_TO_LOCAL = HadoopAuthConfigKeys.HADOOP_SECURITY_AUTH_TO_LOCAL; + public static final String DORIS_KRB5_DEBUG = HadoopAuthConfigKeys.DORIS_KRB5_DEBUG; private static final String DEFAULT_HADOOP_USERNAME = "hadoop"; /** diff --git a/fe/pom.xml b/fe/pom.xml index a04b3ad9838436..5ca799a3266233 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -235,6 +235,7 @@ under the License. fe-extension-spi fe-connector fe-extension-loader + fe-hadoop-runtime fe-sql-parser fe-core hive-udf