From 81d75010c369a0a9784ff16e8a0bbc37e549a57a Mon Sep 17 00:00:00 2001 From: nextdreamblue Date: Thu, 16 Nov 2023 16:55:31 +0800 Subject: [PATCH 1/2] [enhancement](audit) hide password and other sensitive information in log and audit log Signed-off-by: nextdreamblue --- .../analysis/AlterCatalogPropertyStmt.java | 7 +- .../doris/analysis/AlterRepositoryStmt.java | 91 +++++++++++++++++++ .../doris/analysis/AlterResourceStmt.java | 7 +- .../apache/doris/analysis/BrokerLoadStmt.java | 2 +- .../doris/analysis/CreateCatalogStmt.java | 7 +- .../doris/analysis/CreateRepositoryStmt.java | 5 + .../doris/analysis/CreateResourceStmt.java | 7 +- .../apache/doris/analysis/StorageBackend.java | 2 +- .../org/apache/doris/backup/Repository.java | 2 +- .../java/org/apache/doris/catalog/Env.java | 5 +- 10 files changed, 126 insertions(+), 9 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRepositoryStmt.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterCatalogPropertyStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterCatalogPropertyStmt.java index 95480f325f5206..3cac7056276bd0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterCatalogPropertyStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterCatalogPropertyStmt.java @@ -47,6 +47,11 @@ public void analyze(Analyzer analyzer) throws UserException { @Override public String toSql() { return "ALTER CATALOG " + catalogName + " SET PROPERTIES (" - + new PrintableMap<>(newProperties, "=", true, false, ",") + ")"; + + new PrintableMap<>(newProperties, "=", true, false, true) + ")"; + } + + @Override + public boolean needAuditEncryption() { + return true; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRepositoryStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRepositoryStmt.java new file mode 100644 index 00000000000000..c53ee9c00f8d2c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRepositoryStmt.java @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.FeNameFormat; +import org.apache.doris.common.UserException; +import org.apache.doris.common.util.PrintableMap; +import org.apache.doris.datasource.property.constants.S3Properties; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.collect.Maps; + +import java.util.HashMap; +import java.util.Map; + +public class AlterRepositoryStmt extends DdlStmt { + private String name; + private Map properties; + + public AlterRepositoryStmt(String name, Map properties) { + this.name = name; + this.properties = properties; + if (this.properties == null) { + this.properties = Maps.newHashMap(); + } + } + + public Map getProperties() { + return properties; + } + + public String getName() { + return name; + } + + @Override + public void analyze(Analyzer analyzer) throws UserException { + super.analyze(analyzer); + // check auth + if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); + } + FeNameFormat.checkCommonName("repository", name); + Map copyProperties = new HashMap<>(properties); + if (copyProperties.size() == 0) { + throw new UserException("alter repository need contains ak/sk/token info of s3."); + } + copyProperties.remove(S3Properties.ACCESS_KEY); + copyProperties.remove(S3Properties.SECRET_KEY); + copyProperties.remove(S3Properties.SESSION_TOKEN); + copyProperties.remove(S3Properties.Env.ACCESS_KEY); + copyProperties.remove(S3Properties.Env.SECRET_KEY); + copyProperties.remove(S3Properties.Env.TOKEN); + if (copyProperties.size() != 0) { + throw new UserException("alter repository only support ak/sk/token info of s3." + + " unsupported properties: " + copyProperties.keySet()); + } + } + + @Override + public String toSql() { + StringBuilder sb = new StringBuilder(); + sb.append("ALTER REPOSITORY '").append(name).append("' "); + sb.append("PROPERTIES(").append(new PrintableMap<>(properties, " = ", true, false, true)).append(")"); + return sb.toString(); + } + + @Override + public boolean needAuditEncryption() { + return true; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterResourceStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterResourceStmt.java index cd4c1a9fbcc86c..084200f28da17c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterResourceStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterResourceStmt.java @@ -79,7 +79,12 @@ public void analyze(Analyzer analyzer) throws UserException { public String toSql() { StringBuilder sb = new StringBuilder(); sb.append("ALTER RESOURCE '").append(resourceName).append("' "); - sb.append("PROPERTIES(").append(new PrintableMap<>(properties, " = ", true, false)).append(")"); + sb.append("PROPERTIES(").append(new PrintableMap<>(properties, " = ", true, false, true)).append(")"); return sb.toString(); } + + @Override + public boolean needAuditEncryption() { + return true; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerLoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerLoadStmt.java index 64e6a060c19e10..8fb813d5622fad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerLoadStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerLoadStmt.java @@ -117,7 +117,7 @@ public String toSql() { if (properties != null && !properties.isEmpty()) { sb.append("\nPROPERTIES ("); - sb.append(new PrintableMap<>(properties, "=", true, false)); + sb.append(new PrintableMap<>(properties, "=", true, false, true)); sb.append(")"); } return sb.toString(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateCatalogStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateCatalogStmt.java index 7d2bbb2b61d64c..720df2dbedcdf1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateCatalogStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateCatalogStmt.java @@ -122,9 +122,14 @@ public String toSql() { } if (properties.size() > 0) { stringBuilder.append("\nPROPERTIES (\n"); - stringBuilder.append(new PrintableMap<>(properties, "=", true, true, false)); + stringBuilder.append(new PrintableMap<>(properties, "=", true, true, true)); stringBuilder.append("\n)"); } return stringBuilder.toString(); } + + @Override + public boolean needAuditEncryption() { + return true; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRepositoryStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRepositoryStmt.java index 0cb30dd7a36afa..36f4936954a7c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRepositoryStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRepositoryStmt.java @@ -95,4 +95,9 @@ public String toSql() { sb.append("REPOSITORY `").append(name).append("` WITH ").append(storage.toSql()); return sb.toString(); } + + @Override + public boolean needAuditEncryption() { + return true; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java index 6e529be7a9488c..16b7fb99733ebd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java @@ -108,7 +108,12 @@ public String toSql() { sb.append("EXTERNAL "); } sb.append("RESOURCE '").append(resourceName).append("' "); - sb.append("PROPERTIES(").append(new PrintableMap<>(properties, " = ", true, false)).append(")"); + sb.append("PROPERTIES(").append(new PrintableMap<>(properties, " = ", true, false, true)).append(")"); return sb.toString(); } + + @Override + public boolean needAuditEncryption() { + return true; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageBackend.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageBackend.java index e508838207330c..0e148c6c3997ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageBackend.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageBackend.java @@ -132,7 +132,7 @@ public String toSql() { sb.append(" `").append(storageDesc.getName()).append("`"); } sb.append(" ON LOCATION ").append(location).append(" PROPERTIES(") - .append(new PrintableMap<>(storageDesc.getProperties(), " = ", true, false)) + .append(new PrintableMap<>(storageDesc.getProperties(), " = ", true, false, true)) .append(")"); return sb.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java index a95b6a953aa69f..a236ba3c30865f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java @@ -692,7 +692,7 @@ public String getCreateStatement() { stmtBuilder.append("\nPROPERTIES\n("); stmtBuilder.append(new PrintableMap<>(this.getRemoteFileSystem().getProperties(), " = ", - true, true)); + true, true, true)); stmtBuilder.append("\n)"); return stmtBuilder.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 24945649912a04..1f11762cffcac0 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -3363,7 +3363,7 @@ public static void getDdlStmt(DdlStmt ddlStmt, String dbName, TableIf table, Lis sb.append("\nPROPERTIES (\n"); sb.append("\"database\" = \"").append(hiveTable.getHiveDb()).append("\",\n"); sb.append("\"table\" = \"").append(hiveTable.getHiveTable()).append("\",\n"); - sb.append(new PrintableMap<>(hiveTable.getHiveProperties(), " = ", true, true, false).toString()); + sb.append(new PrintableMap<>(hiveTable.getHiveProperties(), " = ", true, true, hidePassword).toString()); sb.append("\n)"); } else if (table.getType() == TableType.ICEBERG) { IcebergTable icebergTable = (IcebergTable) table; @@ -3374,7 +3374,8 @@ public static void getDdlStmt(DdlStmt ddlStmt, String dbName, TableIf table, Lis sb.append("\nPROPERTIES (\n"); sb.append("\"iceberg.database\" = \"").append(icebergTable.getIcebergDb()).append("\",\n"); sb.append("\"iceberg.table\" = \"").append(icebergTable.getIcebergTbl()).append("\",\n"); - sb.append(new PrintableMap<>(icebergTable.getIcebergProperties(), " = ", true, true, false).toString()); + sb.append(new PrintableMap<>(icebergTable.getIcebergProperties(), + " = ", true, true, hidePassword).toString()); sb.append("\n)"); } else if (table.getType() == TableType.JDBC) { JdbcTable jdbcTable = (JdbcTable) table; From f718937eacb01ea1e3ac4ba2e69559c5f7dce193 Mon Sep 17 00:00:00 2001 From: nextdreamblue Date: Thu, 16 Nov 2023 17:30:21 +0800 Subject: [PATCH 2/2] fix code Signed-off-by: nextdreamblue --- .../doris/analysis/AlterRepositoryStmt.java | 91 ------------------- 1 file changed, 91 deletions(-) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRepositoryStmt.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRepositoryStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRepositoryStmt.java deleted file mode 100644 index c53ee9c00f8d2c..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRepositoryStmt.java +++ /dev/null @@ -1,91 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.analysis; - -import org.apache.doris.catalog.Env; -import org.apache.doris.common.ErrorCode; -import org.apache.doris.common.ErrorReport; -import org.apache.doris.common.FeNameFormat; -import org.apache.doris.common.UserException; -import org.apache.doris.common.util.PrintableMap; -import org.apache.doris.datasource.property.constants.S3Properties; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; - -import com.google.common.collect.Maps; - -import java.util.HashMap; -import java.util.Map; - -public class AlterRepositoryStmt extends DdlStmt { - private String name; - private Map properties; - - public AlterRepositoryStmt(String name, Map properties) { - this.name = name; - this.properties = properties; - if (this.properties == null) { - this.properties = Maps.newHashMap(); - } - } - - public Map getProperties() { - return properties; - } - - public String getName() { - return name; - } - - @Override - public void analyze(Analyzer analyzer) throws UserException { - super.analyze(analyzer); - // check auth - if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); - } - FeNameFormat.checkCommonName("repository", name); - Map copyProperties = new HashMap<>(properties); - if (copyProperties.size() == 0) { - throw new UserException("alter repository need contains ak/sk/token info of s3."); - } - copyProperties.remove(S3Properties.ACCESS_KEY); - copyProperties.remove(S3Properties.SECRET_KEY); - copyProperties.remove(S3Properties.SESSION_TOKEN); - copyProperties.remove(S3Properties.Env.ACCESS_KEY); - copyProperties.remove(S3Properties.Env.SECRET_KEY); - copyProperties.remove(S3Properties.Env.TOKEN); - if (copyProperties.size() != 0) { - throw new UserException("alter repository only support ak/sk/token info of s3." - + " unsupported properties: " + copyProperties.keySet()); - } - } - - @Override - public String toSql() { - StringBuilder sb = new StringBuilder(); - sb.append("ALTER REPOSITORY '").append(name).append("' "); - sb.append("PROPERTIES(").append(new PrintableMap<>(properties, " = ", true, false, true)).append(")"); - return sb.toString(); - } - - @Override - public boolean needAuditEncryption() { - return true; - } -}