Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 83 additions & 42 deletions vortex-file/src/strategy.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,74 @@ const ONE_MEG: u64 = 1 << 20;
/// How the compressor was configured on [`WriteStrategyBuilder`].
enum CompressorConfig {
/// A [`BtrBlocksCompressorBuilder`] that [`WriteStrategyBuilder::build`] will finalize.
/// `IntDictScheme` is automatically excluded from the data compressor to prevent recursive
/// dictionary encoding.
/// `IntDictScheme` is automatically excluded when compressing dictionary codes to prevent
/// recursive dictionary encoding.
BtrBlocks(BtrBlocksCompressorBuilder),
/// An opaque compressor used as-is for both data and stats compression.
Opaque(Arc<dyn CompressorPlugin>),
}

/// The compressors that [`WriteStrategyBuilder::build`] derives from a [`CompressorConfig`].
struct Compressors {
/// Compresses dictionary codes.
codes: Arc<dyn CompressorPlugin>,
/// Compresses everything else: chunks that dictionary encoding rejects, dictionary values and
/// stats tables. It also probes columns for dictionary encoding by default.
full: Arc<dyn CompressorPlugin>,
}

impl CompressorConfig {
/// Splits the configured compressor into the codes and full compressors.
///
/// The codes compressor excludes `IntDictScheme` because `DictStrategy` already
/// dictionary-encoded the column, and dictionary-encoding its integer codes again would be
/// redundant. The full compressor keeps every scheme, so chunks that `DictStrategy` rejects can
/// still use integer dictionaries, e.g. for a decimal's most significant part. An opaque
/// compressor is used as-is for both.
fn into_compressors(self) -> Compressors {
match self {
CompressorConfig::BtrBlocks(builder) => Compressors {
codes: Arc::new(
builder
.clone()
.exclude_schemes([IntDictScheme.id()])
.build(),
),
full: Arc::new(builder.build()),
},
CompressorConfig::Opaque(compressor) => Compressors {
codes: Arc::clone(&compressor),
full: compressor,
},
}
}
}

/// Coalesces chunks up to the data block target size, then compresses them with `compressor`
/// and passes them to `buffered`.
fn coalesce_then_compress(
buffered: BufferedStrategy,
compressor: Arc<dyn CompressorPlugin>,
row_block_size: usize,
data_block_target_bytes: Option<u64>,
) -> RepartitionStrategy {
RepartitionStrategy::new(
CompressingStrategy::new(buffered, compressor),
RepartitionWriterOptions {
// Write stream partitions roughly become segments. Because Vortex never reads less
// than one segment, the size of segments and, therefore, partitions, must be small
// enough to both (1) allow fine-grained random access reads and (2) allow
// sufficient read concurrency for the desired throughput. One megabyte is small
// enough to achieve this for S3 (Durner et al., "Exploiting Cloud Object Storage for
// High-Performance Analytics", VLDB Vol 16, Iss 11).
block_size_minimum: data_block_target_bytes.unwrap_or(0),
block_len_multiple: row_block_size,
block_size_target: data_block_target_bytes,
canonicalize: true,
},
)
}

/// Build a new [writer strategy](LayoutStrategy) to compress and reorganize chunks of a Vortex
/// file.
///
Expand Down Expand Up @@ -161,62 +222,42 @@ impl WriteStrategyBuilder {
Arc::new(FlatLayoutStrategy::default())
};

let compressor = self.compressor;

// 7. for each chunk create a flat layout
let chunked = ChunkedLayoutStrategy::new(Arc::clone(&flat));
// 6. buffer chunks so they end up with closer segment ids physically
let buffered = BufferedStrategy::new(chunked, 2 * ONE_MEG); // 2MB

// 5. compress each chunk.
// Exclude IntDictScheme from the data compressor because DictStrategy (step 3) already
// dictionary-encodes columns. Allowing IntDictScheme here would redundantly
// dictionary-encode the integer codes produced by that earlier step.
let data_compressor: Arc<dyn CompressorPlugin> = match &compressor {
CompressorConfig::BtrBlocks(builder) => Arc::new(
builder
.clone()
.exclude_schemes([IntDictScheme.id()])
.build(),
),
CompressorConfig::Opaque(compressor) => Arc::clone(compressor),
};
let compressing = CompressingStrategy::new(buffered, data_compressor);

// 4. prior to compression, coalesce up to a minimum size
let coalescing = RepartitionStrategy::new(
compressing,
RepartitionWriterOptions {
// Write stream partitions roughly become segments. Because Vortex never reads less
// than one segment, the size of segments and, therefore, partitions, must be small
// enough to both (1) allow fine-grained random access reads and (2) allow
// sufficient read concurrency for the desired throughput. One megabyte is small
// enough to achieve this for S3 (Durner et al., "Exploiting Cloud Object Storage for
// High-Performance Analytics", VLDB Vol 16, Iss 11).
block_size_minimum: self.data_block_target_bytes.unwrap_or(0),
block_len_multiple: self.row_block_size,
block_size_target: self.data_block_target_bytes,
canonicalize: true,
},
// 4. coalesce chunks up to a minimum size, then 5. compress each chunk.
// Dict codes and the chunks DictStrategy rejects get separate pipelines because they use
// different compressors; see `CompressorConfig::into_compressors`.
let compressors = self.compressor.into_compressors();
let codes = coalesce_then_compress(
buffered.clone(),
compressors.codes,
self.row_block_size,
self.data_block_target_bytes,
);
let fallback = coalesce_then_compress(
buffered,
Arc::clone(&compressors.full),
self.row_block_size,
self.data_block_target_bytes,
);
let full_compressor = compressors.full;

// 2.1. | 3.1. compress stats tables and dict values.
let stats_compressor: Arc<dyn CompressorPlugin> = match compressor {
CompressorConfig::BtrBlocks(builder) => Arc::new(builder.build()),
CompressorConfig::Opaque(compressor) => compressor,
};
let compress_then_flat = CompressingStrategy::new(flat, Arc::clone(&stats_compressor));
let compress_then_flat = CompressingStrategy::new(flat, Arc::clone(&full_compressor));

// 3. apply dict encoding or fallback
let probe_compressor = if let Some(probe_compressor) = self.probe_compressor {
probe_compressor
} else {
Arc::clone(&stats_compressor)
full_compressor
};
let dict = DictStrategy::new(
coalescing.clone(),
codes,
compress_then_flat.clone(),
coalescing,
fallback,
DictLayoutOptions {
probe_len: self.row_block_size,
..Default::default()
Expand Down
44 changes: 44 additions & 0 deletions vortex-file/src/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -376,6 +376,50 @@ async fn decimal_writer_uses_default_or_session_restrictions(
Ok(())
}

/// A wide decimal with unique values is not dictionary encoded by the dict layout, but its byte
/// parts are still compressed with every scheme, so the low-cardinality most significant part can
/// use an integer dictionary.
#[tokio::test]
#[cfg_attr(miri, ignore)]
async fn rejected_decimal_keeps_nested_int_dict() -> VortexResult<()> {
let distinct = [3i128, -5, 7, 11];
let values = (0..16_384usize)
.map(|i| {
(distinct[(i * 7 + i / 3) % distinct.len()] << 64) + i128::try_from(i * 31).unwrap()
})
.collect::<Buffer<i128>>();
let decimal = DecimalArray::new(values, DecimalDType::new(38, 2), Validity::NonNullable);
let array = StructArray::from_fields(&[("decimal", decimal.into_array())])?.into_array();

let mut buffer = ByteBufferMut::empty();
let summary = SESSION
.write_options()
.disable_editions()
.write(&mut buffer, array.clone().to_array_stream())
.await?;
assert!(!layout_has_dict(summary.footer().layout().as_ref()));

let actual = SESSION
.open_options()
.open_buffer(buffer)?
.scan()?
.into_array_stream()?
.read_all()
.await?;
let has_dict_msp = actual.depth_first_traversal().any(|array| {
array
.as_opt::<DecimalByteParts>()
.is_some_and(|parts| parts.msp().is::<Dict>())
});
assert!(
has_dict_msp,
"expected a dict-encoded msp in {}",
actual.display_tree()
);
assert_arrays_eq!(actual, array, &mut SESSION.create_execution_ctx());
Ok(())
}

#[tokio::test]
#[cfg_attr(miri, ignore)]
async fn test_round_trip_many_types() {
Expand Down
2 changes: 1 addition & 1 deletion vortex-layout/src/layouts/zoned/writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -394,7 +394,7 @@ mod tests {
Nullability::NonNullable,
))]
fn zone_maps_hold_no_variant_state(#[case] dtype: DType) {
let session = SESSION.clone().with::<LayoutSession>();
let session = VortexSession::empty().with::<LayoutSession>();
let accumulator =
AggregateStatsAccumulator::new(&dtype, &default_zoned_aggregate_fns(&dtype, &session));
for aggregate_fn in accumulator.aggregate_fns().iter() {
Expand Down
Loading