Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions be/src/format_v2/table/lance_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,25 @@ Status LanceTableReader::fetch_schema(const TFileRangeDesc& range,
}

namespace {
Status to_lance_approx_mode(TVectorApproxMode::type mode, LanceApproxMode* native_mode) {
switch (mode) {
case TVectorApproxMode::FAST:
*native_mode = LANCE_APPROX_MODE_FAST;
break;
case TVectorApproxMode::NORMAL:
*native_mode = LANCE_APPROX_MODE_NORMAL;
break;
case TVectorApproxMode::ACCURATE:
*native_mode = LANCE_APPROX_MODE_ACCURATE;
break;
default:
return Status::InvalidArgument(
"Lance approx_mode must be FAST, NORMAL, or ACCURATE, got {}",
static_cast<int>(mode));
}
return Status::OK();
}

// A dataset URI without its query string or a password in its user information, either of which
// may carry credentials, for errors and profiles. A user part without a password stays: Azure
// URIs name the container there (abfss://container@account...).
Expand Down Expand Up @@ -619,6 +638,10 @@ Status LanceTableReader::_validate_external_search_request() const {

if (_search_kind == SearchKind::VECTOR && request.__isset.vector_search_options) {
const auto& options = request.vector_search_options;
if (options.__isset.approx_mode) {
LanceApproxMode native_mode;
RETURN_IF_ERROR(to_lance_approx_mode(options.approx_mode, &native_mode));
}
if (options.__isset.query_parallelism && options.query_parallelism < -1) {
return Status::InvalidArgument("Lance query_parallelism must be -1, 0, or positive");
}
Expand Down Expand Up @@ -1131,6 +1154,14 @@ Status LanceTableReader::_configure_vector_search(LanceScanner* scanner,
}
if (request.__isset.vector_search_options) {
const auto& options = request.vector_search_options;
// Leave an omitted mode unset so Lance retains its default search policy.
if (options.__isset.approx_mode) {
LanceApproxMode native_mode;
RETURN_IF_ERROR(to_lance_approx_mode(options.approx_mode, &native_mode));
if (lance_scanner_set_approx_mode(scanner, native_mode) != 0) {
return lance_error("set Lance vector approx_mode");
}
}
if (options.__isset.query_parallelism &&
lance_scanner_set_query_parallelism(scanner, options.query_parallelism) != 0) {
return lance_error("set Lance vector query parallelism");
Expand Down
20 changes: 20 additions & 0 deletions be/test/format_v2/table/lance_reader_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1654,6 +1654,26 @@ TEST(LanceTableReaderVectorSearchTest, RejectsInvalidQueryParallelismBeforeOpeni
EXPECT_NE(std::string::npos, status.to_string().find("query_parallelism"));
}

TEST(LanceTableReaderVectorSearchTest, ValidatesApproxModeBeforeOpeningDataset) {
TQueryGlobals globals;
RuntimeState state(globals);
const Columns columns {projected_column("_distance", TYPE_FLOAT, true)};
for (const int value : {0, 1, 2, 3}) {
SCOPED_TRACE(value);
RuntimeProfile profile("lance_approx_mode_validation");
auto params = make_float32_vector_search_params({0.0F, 0.0F, 0.0F}, 2, 0);
params.lance_scan_params.external_search_request.vector_search_options.__set_approx_mode(
static_cast<TVectorApproxMode::type>(value));
LanceTableReader reader;
const auto status = init_reader(&reader, columns, &state, &profile, &params);
const bool valid = value >= 0 && value <= 2;
EXPECT_EQ(valid, status.ok()) << status.to_string();
if (!valid) {
EXPECT_NE(std::string::npos, status.to_string().find("approx_mode"));
}
}
}

TEST(LanceTableReaderVectorSearchTest, ValidatesDistanceRangeBeforeDatasetAccess) {
TQueryGlobals globals;
RuntimeState state(globals);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,9 @@
import org.apache.doris.thrift.TLanceFileDesc;
import org.apache.doris.thrift.TLanceScanParams;
import org.apache.doris.thrift.TTableFormatFileDesc;
import org.apache.doris.thrift.TVectorApproxMode;
import org.apache.doris.thrift.TVectorMetric;
import org.apache.doris.thrift.TVectorSearchOptions;
import org.apache.doris.thrift.TVectorSearchParams;

import com.google.common.annotations.VisibleForTesting;
Expand Down Expand Up @@ -143,6 +145,7 @@ protected void doInitialize() throws UserException {
super.doInitialize();
checkAdditionalTypeBackendCompatibility(
projectsCurrentReaderType(), backendPolicy.getBackends());
checkApproxModeBackendCompatibility();
ExternalUtil.initSchemaInfo(params, -1L, sourceColumns);

if (searchKind.isExternalSearch()) {
Expand All @@ -157,6 +160,26 @@ public void addLazyMaterializedColumn(String columnName) {
lazyMaterializedColumns.add(columnName.toLowerCase(Locale.ROOT));
}

private void checkApproxModeBackendCompatibility() throws UserException {
if (searchKind != SearchKind.VECTOR || !externalSearchRequest.isSetVectorSearchOptions()) {
return;
}
TVectorSearchOptions options = externalSearchRequest.getVectorSearchOptions();
if (!options.isSetApproxMode() || options.getApproxMode() == TVectorApproxMode.NORMAL) {
return;
}
// Old BEs skip the optional field, which would mix NORMAL with FAST/ACCURATE split candidates.
for (Backend backend : backendPolicy.getBackends()) {
if (backend.isSmoothUpgradeSrc()) {
throw new UserException("Lance approx_mode="
+ options.getApproxMode().name().toLowerCase(Locale.ROOT)
+ " is unavailable while backend " + backend.getId()
+ " is a smooth upgrade source; finish upgrading the query backends"
+ " or use approx_mode=normal");
}
}
}

/** Checks columns read in either phase of a Lance scan. */
private boolean projectsCurrentReaderType() {
// Global row IDs route the second-phase take back to the first-phase BE, so lazy pruning
Expand Down Expand Up @@ -354,6 +377,11 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) {
.append(scanPlan.vectorIndexStatus).append("\n");
result.append(prefix).append("lanceVectorColumn=")
.append(vector.getColumn()).append("\n");
if (externalSearchRequest.isSetVectorSearchOptions()
&& externalSearchRequest.getVectorSearchOptions().isSetApproxMode()) {
result.append(prefix).append("lanceApproxMode=")
.append(externalSearchRequest.getVectorSearchOptions().getApproxMode()).append("\n");
}
if (externalSearchRequest.isSetVectorSearchOptions()
&& externalSearchRequest.getVectorSearchOptions().isSetQueryParallelism()) {
result.append(prefix).append("lanceQueryParallelism=")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.apache.doris.thrift.TExternalSearchQuery;
import org.apache.doris.thrift.TExternalSearchRequest;
import org.apache.doris.thrift.TSearchVector;
import org.apache.doris.thrift.TVectorApproxMode;
import org.apache.doris.thrift.TVectorElementType;
Comment thread
Gabriel39 marked this conversation as resolved.
Resolved
import org.apache.doris.thrift.TVectorMetric;
import org.apache.doris.thrift.TVectorSearchOptions;
Expand Down Expand Up @@ -61,6 +62,7 @@ public class VectorSearchTableValuedFunction extends LanceExternalSearchTableVal
private static final String QUERY_VECTOR = "query_vector";
private static final String METRIC = "metric";
private static final String QUERY_PARALLELISM = "query_parallelism";
private static final String APPROX_MODE = "approx_mode";
private static final String NPROBES = "nprobes";
private static final String REFINE_FACTOR = "refine_factor";
private static final String EF = "ef";
Expand All @@ -69,7 +71,7 @@ public class VectorSearchTableValuedFunction extends LanceExternalSearchTableVal
private static final String USE_INDEX = "use_index";
private static final Set<String> PROPERTIES = ImmutableSet.of(
TABLE, COLUMN, QUERY_VECTOR, TOP_K, OFFSET, METRIC, FILTER, PREFILTER,
NPROBES, REFINE_FACTOR, EF, USE_INDEX, QUERY_PARALLELISM,
NPROBES, REFINE_FACTOR, EF, USE_INDEX, QUERY_PARALLELISM, APPROX_MODE,
VERSION, TIMESTAMP, TAG, BRANCH, DISTANCE_LOWER_BOUND, DISTANCE_UPPER_BOUND);

public VectorSearchTableValuedFunction(Map<String, String> properties)
Expand All @@ -89,6 +91,7 @@ private static PreparedSearch prepare(Map<String, String> properties, boolean de
boolean hasDistanceRange = vectorParams.isSetDistanceLowerBound() || vectorParams.isSetDistanceUpperBound();
boolean useIndex = !params.containsKey(USE_INDEX)
|| parseBoolean(params.get(USE_INDEX), USE_INDEX);
TVectorSearchOptions vectorSearchOptions = buildVectorSearchOptions(params, useIndex);
CommonSearch common = prepareCommon(params, NAME,
"VectorSearchTableValuedFunction", "vector search", useIndex);

Expand Down Expand Up @@ -126,7 +129,6 @@ private static PreparedSearch prepare(Map<String, String> properties, boolean de
TExternalSearchRequest searchRequest = new TExternalSearchRequest()
.setSchemaVersion(1)
.setSearchQuery(TExternalSearchQuery.vector_search(vectorParams));
TVectorSearchOptions vectorSearchOptions = buildVectorSearchOptions(params, useIndex);
if (vectorSearchOptions != null) {
searchRequest.setVectorSearchOptions(vectorSearchOptions);
}
Expand Down Expand Up @@ -169,6 +171,10 @@ static TVectorSearchOptions buildVectorSearchOptions(
Map<String, String> params, boolean useIndex) throws AnalysisException {
TVectorSearchOptions options = new TVectorSearchOptions();
boolean configured = false;
if (params.containsKey(APPROX_MODE)) {
options.setApproxMode(parseApproxMode(params.get(APPROX_MODE)));
configured = true;
}
if (params.containsKey(QUERY_PARALLELISM)) {
options.setQueryParallelism((int) parseLong(
params.get(QUERY_PARALLELISM), QUERY_PARALLELISM, -1, Integer.MAX_VALUE));
Expand All @@ -194,6 +200,22 @@ static TVectorSearchOptions buildVectorSearchOptions(
return configured ? options : null;
}

private static TVectorApproxMode parseApproxMode(String value) throws AnalysisException {
if (value != null) {
switch (value.trim().toLowerCase(Locale.ROOT)) {
case "fast":
return TVectorApproxMode.FAST;
case "normal":
return TVectorApproxMode.NORMAL;
case "accurate":
return TVectorApproxMode.ACCURATE;
default:
break;
}
}
throw new AnalysisException("Unsupported 'approx_mode': expected fast, normal, or accurate");
}

@VisibleForTesting
static List<Column> buildOutputColumns(LanceTableMetadata metadata)
throws AnalysisException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.apache.doris.common.UserException;
import org.apache.doris.common.profile.RuntimeProfile;
import org.apache.doris.common.profile.SummaryProfile;
import org.apache.doris.datasource.ExternalScanNode;
import org.apache.doris.datasource.FederationBackendPolicy;
import org.apache.doris.datasource.lance.LanceExternalCatalog;
import org.apache.doris.datasource.lance.LanceExternalTable;
Expand All @@ -54,6 +55,7 @@
import org.apache.doris.thrift.TFullTextSearchParams;
import org.apache.doris.thrift.TLanceFileDesc;
import org.apache.doris.thrift.TPushAggOp;
import org.apache.doris.thrift.TVectorApproxMode;
import org.apache.doris.thrift.TVectorMetric;
import org.apache.doris.thrift.TVectorSearchOptions;
import org.apache.doris.thrift.TVectorSearchParams;
Expand All @@ -80,6 +82,90 @@

public class LanceScanNodeTest {

@Test
public void testNonDefaultApproxModeRejectsSmoothUpgradeSourcesDuringInitialization() throws Exception {
Backend current = new Backend(1, "127.0.0.1", 9050);
Backend old = new Backend(2, "127.0.0.2", 9050);
old.setSmoothUpgradeSrc(true);
for (TVectorApproxMode mode : Arrays.asList(TVectorApproxMode.FAST, TVectorApproxMode.ACCURATE)) {
for (boolean multiVector : Arrays.asList(false, true)) {
for (List<Backend> backends : Arrays.asList(Collections.singletonList(old),
Arrays.asList(current, old))) {
TExternalSearchRequest request = vectorSearchRequest(5, 0);
request.setVectorSearchOptions(new TVectorSearchOptions().setApproxMode(mode));
if (multiVector) {
request.getSearchQuery().getVectorSearch().setQueryVector(
new org.apache.doris.thrift.TSearchVector()
.setElementType(org.apache.doris.thrift.TVectorElementType.FLOAT32)
.setDimension(2).setNumVectors(2).setValues(new byte[16]));
}
LanceScanNode node = newSearchNode(basicSearchMetadata(vectorSchema()), request);
setBackendCandidates(node, backends);

UserException error = Assert.assertThrows(UserException.class, node::doInitialize);

Assert.assertTrue(error.getMessage(), error.getMessage().contains("approx_mode="
+ mode.name().toLowerCase(java.util.Locale.ROOT)));
Assert.assertTrue(error.getMessage(), error.getMessage().contains("backend 2"));
Assert.assertTrue(error.getMessage(), error.getMessage().contains("smooth upgrade source"));
}
}
}
}

@Test
public void testAllApproxModesInitializeWithCurrentBackends() throws Exception {
List<Backend> backends = Arrays.asList(new Backend(1, "127.0.0.1", 9050),
new Backend(2, "127.0.0.2", 9050));
for (TVectorApproxMode mode : TVectorApproxMode.values()) {
TExternalSearchRequest request = vectorSearchRequest(5, 0);
request.setVectorSearchOptions(new TVectorSearchOptions().setApproxMode(mode));
LanceScanNode node = newSearchNode(basicSearchMetadata(vectorSchema()), request);
setBackendCandidates(node, backends);

node.doInitialize();

Assert.assertEquals(mode, node.createSplitSearchRequest().getVectorSearchOptions().getApproxMode());
}
}

@Test
public void testDefaultApproxModeRemainsCompatibleWithSmoothUpgradeSources() throws Exception {
Backend old = new Backend(2, "127.0.0.2", 9050);
old.setSmoothUpgradeSrc(true);
List<Backend> backends = Arrays.asList(new Backend(1, "127.0.0.1", 9050), old);
for (TVectorSearchOptions options : Arrays.asList(null, new TVectorSearchOptions().setNprobes(4),
new TVectorSearchOptions().setApproxMode(TVectorApproxMode.NORMAL))) {
TExternalSearchRequest request = vectorSearchRequest(5, 0);
if (options != null) {
request.setVectorSearchOptions(options);
}
LanceScanNode node = newSearchNode(basicSearchMetadata(vectorSchema()), request);
setBackendCandidates(node, backends);

node.doInitialize();

TExternalSearchRequest split = node.createSplitSearchRequest();
Assert.assertEquals(request.isSetVectorSearchOptions(), split.isSetVectorSearchOptions());
if (options != null) {
Assert.assertEquals(options, split.getVectorSearchOptions());
}
}
}

@Test
public void testApproxModeCompatibilityDoesNotRejectFullTextSearch() throws Exception {
Backend old = new Backend(2, "127.0.0.2", 9050);
old.setSmoothUpgradeSrc(true);
TExternalSearchRequest request = fullTextSearchRequest(5, 0, TFtsCoverageMode.STRICT);
LanceScanNode node = newSearchNode(basicSearchMetadata(fullTextSchema()), request);
setBackendCandidates(node, Collections.singletonList(old));

node.doInitialize();

Assert.assertEquals(request, node.createSplitSearchRequest());
}

@Test
public void testSplitPlanningReportsSuccessAndFailureToQueryProfile() throws Exception {
ConnectContext previous = ConnectContext.get();
Expand Down Expand Up @@ -669,13 +755,16 @@ public void testExternalSearchUsesOneSplitPerIndexSegmentAndKeepsUnindexedFragme
Collections.singletonList(9), Arrays.asList(3L, 4L),
IndexType.VECTOR, "L2")));
TExternalSearchRequest request = vectorSearchRequest(5, 0);
request.setVectorSearchOptions(new TVectorSearchOptions().setQueryParallelism(4));
request.setVectorSearchOptions(new TVectorSearchOptions()
.setQueryParallelism(4).setApproxMode(TVectorApproxMode.ACCURATE));
LanceScanNode node = newSearchNode(metadata, request);

List<Split> splits = node.getSplits(3);

Assert.assertTrue(node.getNodeExplainString("", TExplainLevel.NORMAL)
.contains("lanceQueryParallelism=4"));
Assert.assertTrue(node.getNodeExplainString("", TExplainLevel.NORMAL)
.contains("lanceApproxMode=ACCURATE"));
Assert.assertTrue(node.getNodeExplainString("", TExplainLevel.NORMAL)
.contains("lanceVectorIndexStatus=USED"));

Expand Down Expand Up @@ -1110,6 +1199,21 @@ private static LanceScanNode newNode() {
return newNode(new SessionVariable());
}

private static LanceTableMetadata basicSearchMetadata(Schema schema) {
return LanceTableMetadata.createBasicSnapshot(
new LanceTableAccess("s3://bucket/table.lance", Collections.emptyMap()), 42, schema,
Collections.singletonList(new LanceFragmentInfo(1, 8, 8)));
}

private static void setBackendCandidates(LanceScanNode node, List<Backend> backends) throws Exception {
FederationBackendPolicy policy = Mockito.mock(FederationBackendPolicy.class);
Mockito.when(policy.getBackends()).thenReturn(backends);
Mockito.when(policy.numBackends()).thenReturn(backends.size());
java.lang.reflect.Field field = ExternalScanNode.class.getDeclaredField("backendPolicy");
field.setAccessible(true);
field.set(node, policy);
}

private static LanceScanNode newNode(SessionVariable sessionVariable) {
return new LanceScanNode(
new PlanNodeId(0),
Expand Down
Loading
Loading