diff --git a/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl index 774c83a5db380a..ea267a4edeafd2 100644 --- a/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl +++ b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl @@ -35,8 +35,8 @@ services: - ./spark-defaults.conf:/opt/spark/conf/spark-defaults.conf - ./data/input/jars/iceberg-aws-bundle-1.10.1.jar:/opt/spark/jars/iceberg-aws-bundle-1.10.1.jar - ./data/input/jars/iceberg-spark-runtime-4.0_2.13-1.10.1.jar:/opt/spark/jars/iceberg-spark-runtime-4.0_2.13-1.10.1.jar - - ./data/input/jars/paimon-s3-1.3.1.jar:/opt/spark/jars/paimon-s3-1.3.1.jar - - ./data/input/jars/paimon-spark-4.0-1.3.1.jar:/opt/spark/jars/paimon-spark-4.0-1.3.1.jar + - ./data/input/jars/paimon-s3-1.4.2.jar:/opt/spark/jars/paimon-s3-1.4.2.jar + - ./data/input/jars/paimon-spark-4.0_2.13-1.4.2.jar:/opt/spark/jars/paimon-spark-4.0_2.13-1.4.2.jar environment: - AWS_ACCESS_KEY_ID=admin - AWS_SECRET_ACCESS_KEY=password diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run13.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run13.sql index 37ac16ac3d1645..ec071966382076 100644 --- a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run13.sql +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run13.sql @@ -63,8 +63,10 @@ tblproperties ( insert into variant_mixed_su values (1, date '2026-06-01', parse_json('{"name":"alice","age":18,"layout":"shredded"}')); -alter table variant_mixed_su set tblproperties ( - 'parquet.variant.shreddingSchema' = '' +-- Paimon 1.4.2 parses every present shredding schema as JSON, so remove the property +-- before writing unshredded files. +alter table variant_mixed_su unset tblproperties ( + 'parquet.variant.shreddingSchema' ); insert into variant_mixed_su values (2, date '2026-07-01', parse_json('{"name":"bob","age":30,"layout":"unshredded"}')); diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh index fb2d8bb4f3ad25..0f83913380b2de 100755 --- a/docker/thirdparties/run-thirdparties-docker.sh +++ b/docker/thirdparties/run-thirdparties-docker.sh @@ -1437,21 +1437,34 @@ start_iceberg() { register_stack_metadata "iceberg" "${ROOT}/docker-compose/iceberg/iceberg.yaml" "${ROOT}/docker-compose/iceberg/iceberg.env" compose_down_stack "${ROOT}/docker-compose/iceberg/iceberg.yaml" "${ROOT}/docker-compose/iceberg/iceberg.env" --remove-orphans if [[ "${STOP}" -ne 1 ]]; then + # data/ is extracted from the fixture archive once and then reused. A data/ from an older archive lacks + # jars that iceberg.yaml now mounts, and Compose would create empty directories in their place, so + # start over from the current archive. + if [[ -d "${ICEBERG_DIR}/data" ]]; then + local jar + for jar in $(grep -o '\./data/input/jars/[^:]*\.jar' "${ICEBERG_DIR}/iceberg.yaml"); do + if [[ ! -f "${ICEBERG_DIR}/${jar#./}" ]]; then + echo "${ICEBERG_DIR}/${jar#./} is missing, ${ICEBERG_DIR}/data is from an older archive" + host_admin_cmd rm -rf "${ICEBERG_DIR:?}/data" + break + fi + done + fi if [[ ! -d "${ICEBERG_DIR}/data" ]]; then echo "${ICEBERG_DIR}/data does not exist" ( cd "${ICEBERG_DIR}" || exit 1 rm -f iceberg_data*.zip if [[ "${HOST_OS}" == "Darwin" ]]; then - curl -fL -o iceberg_data_spark40.zip \ - "https://${s3BucketName}.${s3Endpoint}/regression/datalake/pipeline_data/iceberg_data_spark40.zip" + curl -fL -o iceberg_data_spark40_paimon142.zip \ + "https://${s3BucketName}.${s3Endpoint}/regression/datalake/pipeline_data/iceberg_data_spark40_paimon142.zip" else wget -P "${ROOT}/docker-compose/iceberg" \ - "https://${s3BucketName}.${s3Endpoint}/regression/datalake/pipeline_data/iceberg_data_spark40.zip" + "https://${s3BucketName}.${s3Endpoint}/regression/datalake/pipeline_data/iceberg_data_spark40_paimon142.zip" fi - host_admin_cmd unzip iceberg_data_spark40.zip + host_admin_cmd unzip iceberg_data_spark40_paimon142.zip host_admin_cmd mv iceberg_data data - host_admin_cmd rm -rf iceberg_data_spark40.zip + host_admin_cmd rm -rf iceberg_data_spark40_paimon142.zip ) else echo "${ICEBERG_DIR}/data exist, continue !" diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_partition_table.out b/regression-test/data/external_table_p0/paimon/test_paimon_partition_table.out index 3230ad01841d21..4da4e78a3fff97 100644 --- a/regression-test/data/external_table_p0/paimon/test_paimon_partition_table.out +++ b/regression-test/data/external_table_p0/paimon/test_paimon_partition_table.out @@ -1,30 +1,30 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !show_partition_sales_by_date -- -sale_date=2024-01-15 sale_date 2 2216 1 -sale_date=2024-01-16 sale_date 2 4217 2 -sale_date=2024-01-17 sale_date 1 2118 1 +sale_date=2024-01-15 sale_date 2 2100 1 +sale_date=2024-01-16 sale_date 2 3985 2 +sale_date=2024-01-17 sale_date 1 2002 1 -- !show_partition_sales_by_region -- -region=China-Beijing region 1 2425 1 -region=Japan-Tokyo region 1 2420 1 -region=USA-California region 1 2454 1 +region=China-Beijing region 1 2295 1 +region=Japan-Tokyo region 1 2290 1 +region=USA-California region 1 2324 1 -- !show_partition_sales_by_date_region -- -sale_date=2024-01-15/region=China-Beijing sale_date,region 1 2627 1 -sale_date=2024-01-15/region=Japan-Tokyo sale_date,region 1 2614 1 -sale_date=2024-01-15/region=USA-California sale_date,region 1 2655 1 -sale_date=2024-01-16/region=China-Shanghai sale_date,region 1 2636 1 -sale_date=2024-01-16/region=Japan-Osaka sale_date,region 1 2636 1 -sale_date=2024-01-16/region=USA-New York sale_date,region 1 2643 1 +sale_date=2024-01-15/region=China-Beijing sale_date,region 1 2485 1 +sale_date=2024-01-15/region=Japan-Tokyo sale_date,region 1 2472 1 +sale_date=2024-01-15/region=USA-California sale_date,region 1 2513 1 +sale_date=2024-01-16/region=China-Shanghai sale_date,region 1 2494 1 +sale_date=2024-01-16/region=Japan-Osaka sale_date,region 1 2494 1 +sale_date=2024-01-16/region=USA-New York sale_date,region 1 2501 1 -- !show_partition_events_by_hour -- -hour_partition=2024-01-15-10 hour_partition 2 2361 1 -hour_partition=2024-01-15-11 hour_partition 2 4515 2 -hour_partition=2024-01-15-14 hour_partition 2 2371 1 +hour_partition=2024-01-15-10 hour_partition 2 2245 1 +hour_partition=2024-01-15-11 hour_partition 2 4283 2 +hour_partition=2024-01-15-14 hour_partition 2 2255 1 -- !show_partition_logs_by_date_hierarchy -- -year_val=2024/month_val=1/day_val=15 year_val,month_val,day_val 2 2841 1 -year_val=2024/month_val=1/day_val=16 year_val,month_val,day_val 2 5323 2 -year_val=2024/month_val=1/day_val=17 year_val,month_val,day_val 1 2658 1 -year_val=2024/month_val=2/day_val=1 year_val,month_val,day_val 1 2686 1 +year_val=2024/month_val=1/day_val=15 year_val,month_val,day_val 2 2701 1 +year_val=2024/month_val=1/day_val=16 year_val,month_val,day_val 2 5042 2 +year_val=2024/month_val=1/day_val=17 year_val,month_val,day_val 1 2517 1 +year_val=2024/month_val=2/day_val=1 year_val,month_val,day_val 1 2546 1 diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_external_paths.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_external_paths.out index 136ed02abc086b..a7208f2c81506a 100644 --- a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_external_paths.out +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_external_paths.out @@ -32,9 +32,18 @@ p2 4 4 p3 5 4 p3 6 3 +-- !external_weight_robin -- +1 weight-1 +2 weight-2 +3 weight-3 +4 weight-4 +5 weight-5 +6 weight-6 + -- !external_specific_fs -- 1 specific-1 2 specific-2 -- !external_default_path -- 1 default-path + diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_external_paths.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_external_paths.groovy index 6077175cb292d4..d12b3e3a751703 100644 --- a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_external_paths.groovy +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_external_paths.groovy @@ -59,9 +59,19 @@ suite("test_paimon_write_external_paths", "p0,external,paimon") { 'data-file.external-paths.strategy' = 'round-robin' ); - -- The Spark fixture uses Paimon 1.3.1, which cannot create a table with the - -- weight-robin strategy introduced in Paimon 1.4. Re-enable that strategy - -- after the fixture has a compatible Spark connector. + DROP TABLE IF EXISTS paimon.${dbName}.t_weight_robin; + CREATE TABLE paimon.${dbName}.t_weight_robin ( + id INT, payload STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'write-only' = 'true', + 'target-file-size' = '1 kb', + 'data-file.external-paths' = '${pathRoot}/weight-a,${pathRoot}/weight-b', + 'data-file.external-paths.strategy' = 'weight-robin', + 'data-file.external-paths.weights' = '1,1' + ); DROP TABLE IF EXISTS paimon.${dbName}.t_specific_fs; CREATE TABLE paimon.${dbName}.t_specific_fs ( @@ -197,6 +207,18 @@ suite("test_paimon_write_external_paths", "p0,external,paimon") { assertDorisSparkRows("external_round_robin_changed", "t_round_robin", "pt, id, length(payload)", "ORDER BY pt, id") + (1..6).each { id -> + sql """INSERT INTO t_weight_robin VALUES (${id}, 'weight-${id}')""" + } + def weightedFiles = dataFiles("t_weight_robin") + assertFalse(weightedFiles.isEmpty()) + assertTrue(weightedFiles.every { + it.startsWith("${pathRoot}/weight-a/") || + it.startsWith("${pathRoot}/weight-b/") + }) + assertDorisSparkRows("external_weight_robin", "t_weight_robin", + "id, payload", "ORDER BY id") + sql """INSERT INTO t_specific_fs VALUES (1, 'specific-1')""" sql """INSERT INTO t_specific_fs VALUES (2, 'specific-2')""" def specificFiles = dataFiles("t_specific_fs")