Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,10 @@ public String toSql(int depth) {
}

sb.append("PROPERTIES (");
if (enableVariantDocMode) {
if (enableNestedGroup) {
// A nested group column has no subcolumn or doc mode settings, the parser clears them.
sb.append("\"variant_enable_nested_group\" = \"true\"");
} else if (enableVariantDocMode) {
sb.append("\"variant_enable_doc_mode\" = \"")
.append(String.valueOf(enableVariantDocMode)).append("\"");
sb.append(",");
Expand All @@ -219,11 +222,6 @@ public String toSql(int depth) {
sb.append("\"variant_sparse_hash_shard_count\" = \"")
.append(String.valueOf(Math.max(1, variantSparseHashShardCount))).append("\"");
}
if (enableNestedGroup) {
sb.append(",");
sb.append("\"variant_enable_nested_group\" = \"")
.append(String.valueOf(enableNestedGroup)).append("\"");
}
sb.append(")>");
return sb.toString();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1111,6 +1111,7 @@
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ConnectContextUtil;
import org.apache.doris.qe.GlobalVariable;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.qe.SqlModeHelper;
import org.apache.doris.resource.workloadschedpolicy.WorkloadActionMeta;
import org.apache.doris.resource.workloadschedpolicy.WorkloadConditionMeta;
Expand Down Expand Up @@ -1166,6 +1167,8 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {

private static String DEFAULT_NESTED_COLUMN_NAME = "unnest";
private static String DEFAULT_ORDINALITY_COLUMN_NAME = "ordinality";
// The built-in values of the default_variant_* session variables.
private static final SessionVariable BUILT_IN_SESSION_VARIABLE = new SessionVariable();

private static class ColumnDefinitionWithPath {
private final ColumnDefinition columnDefinition;
Expand Down Expand Up @@ -5400,25 +5403,25 @@ public DataType visitVariantPredefinedFields(VariantPredefinedFieldsContext ctx)
Map<String, String> properties = variantCtx.properties != null
? Maps.newHashMap(visitPropertyClause(variantCtx.properties)) : Maps.newHashMap();

int variantMaxSubcolumnsCount = ConnectContext.get() == null ? 0 :
ConnectContext.get().getSessionVariable().getDefaultVariantMaxSubcolumnsCount();
boolean enableTypedPathsToSparse = ConnectContext.get() == null ? false :
ConnectContext.get().getSessionVariable().getDefaultEnableTypedPathsToSparse();
int variantMaxSparseColumnStatisticsSize = ConnectContext.get() == null ? 0 :
ConnectContext.get().getSessionVariable().getDefaultVariantMaxSparseColumnStatisticsSize();
// The default_variant_* session variables only drive the tests, which run with debug points enabled.
// Elsewhere, and without a session, the properties a DDL leaves out take the built-in values, whatever the
// session holds (a global value set before the variables became test-only still loads from an image), so a
// DDL text, such as the one CREATE TABLE LIKE prints and parses, creates the same column in every session.
SessionVariable variantDefaults = BUILT_IN_SESSION_VARIABLE;
if (Config.enable_debug_points && ConnectContext.get() != null) {
variantDefaults = ConnectContext.get().getSessionVariable();
}
int variantMaxSubcolumnsCount = variantDefaults.getDefaultVariantMaxSubcolumnsCount();
boolean enableTypedPathsToSparse = variantDefaults.getDefaultEnableTypedPathsToSparse();
int variantMaxSparseColumnStatisticsSize = variantDefaults.getDefaultVariantMaxSparseColumnStatisticsSize();
// default no bucketing
int variantSparseHashShardCount = ConnectContext.get() == null ? 0 :
ConnectContext.get().getSessionVariable().getDefaultVariantSparseHashShardCount();

boolean enableVariantDocMode = ConnectContext.get() == null ? false :
ConnectContext.get().getSessionVariable().getDefaultVariantEnableDocMode();
long variantDocMaterializationMinRows = ConnectContext.get() == null ? 0L :
ConnectContext.get().getSessionVariable().getDefaultVariantDocMaterializationMinRows();
int variantDocHashShardCount = ConnectContext.get() == null ? 128 :
ConnectContext.get().getSessionVariable().getDefaultVariantDocHashShardCount();

boolean enableNestedGroup = ConnectContext.get() == null ? true :
ConnectContext.get().getSessionVariable().getDefaultVariantEnableNestedGroup();
int variantSparseHashShardCount = variantDefaults.getDefaultVariantSparseHashShardCount();

boolean enableVariantDocMode = variantDefaults.getDefaultVariantEnableDocMode();
long variantDocMaterializationMinRows = variantDefaults.getDefaultVariantDocMaterializationMinRows();
int variantDocHashShardCount = variantDefaults.getDefaultVariantDocHashShardCount();

boolean enableNestedGroup = variantDefaults.getDefaultVariantEnableNestedGroup();

try {
// validate properties: variant_enable_doc_mode cannot be set together with other properties
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,10 @@ public String toSql() {
}

sb.append("PROPERTIES (");
if (enableVariantDocMode) {
if (enableNestedGroup) {
// A nested group column has no subcolumn or doc mode settings, the parser clears them.
sb.append("\"variant_enable_nested_group\" = \"true\"");
} else if (enableVariantDocMode) {
sb.append("\"variant_enable_doc_mode\" = \"")
.append(String.valueOf(enableVariantDocMode)).append("\"");
sb.append(",");
Expand All @@ -229,11 +232,6 @@ public String toSql() {
.append(String.valueOf(Math.max(1, variantSparseHashShardCount)))
.append("\"");
}
if (enableNestedGroup) {
sb.append(",");
sb.append("\"variant_enable_nested_group\" = \"")
.append(String.valueOf(enableNestedGroup)).append("\"");
}
sb.append(")>");
return sb.toString();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3901,6 +3901,7 @@ public void checkAnnIndexCandidateRowsPercentThreshold(String value) {

@VariableMgr.VarAttr(
name = DEFAULT_VARIANT_MAX_SUBCOLUMNS_COUNT,
flag = VariableMgr.INVISIBLE,
needForward = true,
checker = "checkDefaultVariantMaxSubcolumnsCount",
fuzzy = true
Expand All @@ -3909,13 +3910,16 @@ public void checkAnnIndexCandidateRowsPercentThreshold(String value) {

@VariableMgr.VarAttr(
name = DEFAULT_VARIANT_ENABLE_TYPED_PATHS_TO_SPARSE,
flag = VariableMgr.INVISIBLE,
needForward = true,
checker = "checkDefaultVariantSettable",
fuzzy = true
)
public boolean defaultEnableTypedPathsToSparse = false;

@VariableMgr.VarAttr(
name = DEFAULT_VARIANT_MAX_SPARSE_COLUMN_STATISTICS_SIZE,
flag = VariableMgr.INVISIBLE,
needForward = true,
checker = "checkDefaultVariantMaxSparseColumnStatisticsSize",
fuzzy = true
Expand All @@ -3933,7 +3937,9 @@ public void checkAnnIndexCandidateRowsPercentThreshold(String value) {

@VariableMgr.VarAttr(
name = DEFAULT_VARIANT_SPARSE_HASH_SHARD_COUNT,
flag = VariableMgr.INVISIBLE,
needForward = true,
checker = "checkDefaultVariantSettable",
fuzzy = true
)
public int defaultVariantSparseHashShardCount = 0;
Expand All @@ -3946,29 +3952,37 @@ public void checkAnnIndexCandidateRowsPercentThreshold(String value) {

@VariableMgr.VarAttr(
name = DEFAULT_VARIANT_ENABLE_DOC_MODE,
flag = VariableMgr.INVISIBLE,
needForward = true,
checker = "checkDefaultVariantSettable",
fuzzy = true
)
public boolean defaultVariantEnableDocMode = false;


@VariableMgr.VarAttr(
name = DEFAULT_VARIANT_DOC_MATERIALIZATION_MIN_ROWS,
flag = VariableMgr.INVISIBLE,
needForward = true,
checker = "checkDefaultVariantSettable",
fuzzy = true
)
public long defaultVariantDocMaterializationMinRows = 0L;

@VariableMgr.VarAttr(
name = DEFAULT_VARIANT_DOC_HASH_SHARD_COUNT,
flag = VariableMgr.INVISIBLE,
needForward = true,
checker = "checkDefaultVariantSettable",
fuzzy = true
)
public int defaultVariantDocHashShardCount = 64;

@VariableMgr.VarAttr(
name = DEFAULT_VARIANT_ENABLE_NESTED_GROUP,
flag = VariableMgr.INVISIBLE,
needForward = true,
checker = "checkDefaultVariantSettable",
fuzzy = true
)
public boolean defaultVariantEnableNestedGroup = false;
Expand Down Expand Up @@ -6994,7 +7008,17 @@ public static boolean enableStrictCast() {
}
}

// The default_variant_* variables only drive the tests, which run with debug points enabled. Users state VARIANT
// properties in the column definition, so a DDL text creates the same column in every session.
public void checkDefaultVariantSettable(String value) {
if (!Config.enable_debug_points) {
throw new UnsupportedOperationException("default_variant_* session variables are only for testing, "
+ "set the property in the VARIANT column definition");
}
}

public void checkDefaultVariantMaxSubcolumnsCount(String variantMaxSubcolumnsCount) {
checkDefaultVariantSettable(variantMaxSubcolumnsCount);
int value = Integer.valueOf(variantMaxSubcolumnsCount);
if (value < 0 || value > 100000) {
throw new UnsupportedOperationException(
Expand All @@ -7003,6 +7027,7 @@ public void checkDefaultVariantMaxSubcolumnsCount(String variantMaxSubcolumnsCou
}

public void checkDefaultVariantMaxSparseColumnStatisticsSize(String variantMaxSparseColumnStatisticsSize) {
checkDefaultVariantSettable(variantMaxSparseColumnStatisticsSize);
int value = Integer.valueOf(variantMaxSparseColumnStatisticsSize);
if (value < 1 || value > 50000) {
throw new UnsupportedOperationException("variant max sparse column statistics size is: "
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.apache.doris.catalog;

import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.FeConstants;
Expand All @@ -26,6 +27,7 @@
import org.apache.doris.nereids.trees.plans.commands.CreateTableLikeCommand;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.qe.StmtExecutor;
import org.apache.doris.utframe.TestWithFeService;
import org.apache.doris.utframe.UtFrameUtils;
Expand Down Expand Up @@ -89,6 +91,44 @@ private static void createTable(String sql) throws Exception {
}
}

@Test
public void testVariantTypeIgnoresSessionDefaults() throws Exception {
SessionVariable sessionVariable = connectContext.getSessionVariable();
boolean originalEnableDebugPoints = Config.enable_debug_points;
boolean originalDocMode = sessionVariable.defaultVariantEnableDocMode;
boolean originalNestedGroup = sessionVariable.defaultVariantEnableNestedGroup;
try {
Config.enable_debug_points = false;
createTable("CREATE TABLE test.variant_like_src (id INT, "
+ "v VARIANT<'a.b':INT, PROPERTIES(\"variant_max_subcolumns_count\"=\"3\")>, "
+ "z VARIANT<PROPERTIES(\"variant_max_subcolumns_count\"=\"0\")>, "
+ "d VARIANT<PROPERTIES(\"variant_enable_doc_mode\"=\"true\")>, "
+ "n VARIANT<PROPERTIES(\"variant_enable_nested_group\"=\"true\")>) "
+ "DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES(\"replication_num\"=\"1\")");
Database db = Env.getCurrentInternalCatalog().getDbOrDdlException("test");
Table source = db.getTableOrDdlException("variant_like_src");
String sourceDdl = getCreateTableStmt(source);
// CREATE TABLE LIKE parses the DDL of the source, which prints neither a disabled doc mode nor a
// disabled nested group. Without debug points the session variables do not fill them in.
for (boolean docMode : new boolean[] {true, false}) {
sessionVariable.defaultVariantEnableDocMode = docMode;
sessionVariable.defaultVariantEnableNestedGroup = !docMode;
String targetName = docMode ? "variant_like_doc_mode" : "variant_like_nested_group";
createTableLike("CREATE TABLE test." + targetName + " LIKE test.variant_like_src");
Table target = db.getTableOrDdlException(targetName);
for (String columnName : new String[] {"v", "z", "d", "n"}) {
Assert.assertEquals(source.getColumn(columnName).getType(),
target.getColumn(columnName).getType());
}
Assert.assertEquals(sourceDdl.replace("variant_like_src", targetName), getCreateTableStmt(target));
}
} finally {
Config.enable_debug_points = originalEnableDebugPoints;
sessionVariable.defaultVariantEnableDocMode = originalDocMode;
sessionVariable.defaultVariantEnableNestedGroup = originalNestedGroup;
}
}

private static void createTableLike(String sql) throws Exception {
NereidsParser nereidsParser = new NereidsParser();
CreateTableLikeCommand command = (CreateTableLikeCommand) nereidsParser.parseSingle(sql);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import org.apache.doris.common.Config;
import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.qe.SessionVariable;

import org.junit.Assert;
import org.junit.Test;
Expand Down Expand Up @@ -172,7 +173,10 @@ public void testVariantToSqlSerializesNestedGroupProperty() {
VariantType variantType = new VariantType(new ArrayList<>(), 0, false, 10000, 0,
false, 0L, 64, true);

Assert.assertTrue(variantType.toSql().contains("\"variant_enable_nested_group\" = \"true\""));
// Only the nested group property: the others are cleared for nested group and some, such as a statistics
// size of 0, would not parse.
Assert.assertEquals("variant<PROPERTIES (\"variant_enable_nested_group\" = \"true\")>",
variantType.toSql());
}

@Test
Expand Down Expand Up @@ -206,6 +210,20 @@ public void testVariantFieldCommentToSqlRoundTrip() {
Assert.assertEquals(comment, reparsed.getPredefinedFields().get(0).getComment());
}

@Test
public void testVariantTakesBuiltInDefaults() {
// Without debug points, and without a session, the properties a DDL leaves out take the built-in values
// of the default_variant_* session variables.
org.apache.doris.nereids.types.VariantType parsed = (org.apache.doris.nereids.types.VariantType)
new NereidsParser().parseDataType("variant");
SessionVariable builtIn = new SessionVariable();
Assert.assertEquals(builtIn.defaultVariantMaxSubcolumnsCount, parsed.getVariantMaxSubcolumnsCount());
Assert.assertEquals(builtIn.defaultVariantMaxSparseColumnStatisticsSize,
parsed.getVariantMaxSparseColumnStatisticsSize());
Assert.assertEquals(builtIn.defaultVariantEnableDocMode, parsed.getEnableVariantDocMode());
Assert.assertEquals(builtIn.defaultVariantEnableNestedGroup, parsed.getEnableNestedGroup());
}

// ===================== Mixed Nesting & Precision =====================
@Test
public void testArrayMapStructCombinationWithPrecision() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -474,9 +474,17 @@ public void testAnalyzeVariantMaxSparseColumnStatisticsSize() throws AnalysisExc

@Test
public void testCheckDefaultVariantMaxSparseColumnStatisticsSize() {
SessionVariable sessionVariable = new SessionVariable();
Assertions.assertThrows(UnsupportedOperationException.class,
() -> sessionVariable.checkDefaultVariantMaxSparseColumnStatisticsSize("0"));
sessionVariable.checkDefaultVariantMaxSparseColumnStatisticsSize("1");
// The default_variant_* variables can only be set with debug points enabled.
boolean originalEnableDebugPoints = Config.enable_debug_points;
try {
Config.enable_debug_points = true;
SessionVariable sessionVariable = new SessionVariable();
UnsupportedOperationException e = Assertions.assertThrows(UnsupportedOperationException.class,
() -> sessionVariable.checkDefaultVariantMaxSparseColumnStatisticsSize("0"));
Assertions.assertTrue(e.getMessage().contains("it must between 1 and 50000"), e.getMessage());
sessionVariable.checkDefaultVariantMaxSparseColumnStatisticsSize("1");
} finally {
Config.enable_debug_points = originalEnableDebugPoints;
}
}
}
Loading
Loading