chrevanthreddy commented on code in PR #19310: URL: https://github.com/apache/hudi/pull/19310#discussion_r4149332484
########## hudi-common/src/main/java/org/apache/hudi/common/index/vector/VectorIndexOptions.java: ########## @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.common.index.vector; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * Options accepted by {@code CREATE INDEX ... USING VECTOR}. + * + * <p>The indexed column's Hudi {@code VECTOR(D[, elementType])} schema is authoritative for + * dimension and element type. Index options configure only the acceleration structure. DDL + * implementations must call {@link #validateAndNormalize(Map)} before persisting an index + * definition; individual parsing helpers are intentionally private so aggregate validation cannot + * be bypassed. + */ +public final class VectorIndexOptions { + + public static final String METRIC = "vector.metric"; + public static final String QUANTIZER = "vector.quantizer"; + public static final String NUM_CLUSTERS = "vector.num_clusters"; + public static final String MAX_ITER = "vector.max_iter"; + public static final String RABITQ_BITS = "vector.rabitq.bits"; + public static final String RABITQ_SEED = "vector.rabitq.seed"; + public static final String RABITQ_ASSUME_NORMALIZED = "vector.rabitq.assume_normalized"; + public static final String QUERY_NUM_PROBES = "vector.query.nprobes"; + public static final String QUERY_REFINE_FACTOR = "vector.query.refine_factor"; + public static final String QUERY_MODE = "vector.query.mode"; + public static final String QUERY_STALE_POLICY = "vector.query.stale_policy"; + + public static final VectorDistanceMetric DEFAULT_METRIC = VectorDistanceMetric.COSINE; + public static final VectorQuantizer DEFAULT_QUANTIZER = VectorQuantizer.IVF_RABITQ; + public static final int DEFAULT_NUM_CLUSTERS = 256; + public static final int DEFAULT_MAX_ITER = 20; + public static final int DEFAULT_RABITQ_BITS = 4; + public static final long DEFAULT_RABITQ_SEED = 42L; + public static final int DEFAULT_NUM_PROBES = 32; + public static final int DEFAULT_REFINE_FACTOR = 50; + public static final VectorQueryMode DEFAULT_QUERY_MODE = VectorQueryMode.EXACT_RERANK; + public static final VectorStalePolicy DEFAULT_STALE_POLICY = VectorStalePolicy.FAIL; + + private static final Set<String> SUPPORTED_OPTIONS = Collections.unmodifiableSet( + new HashSet<>(Arrays.asList( + METRIC, + QUANTIZER, + NUM_CLUSTERS, + MAX_ITER, + RABITQ_BITS, + RABITQ_SEED, + RABITQ_ASSUME_NORMALIZED, + QUERY_NUM_PROBES, + QUERY_REFINE_FACTOR, + QUERY_MODE, + QUERY_STALE_POLICY))); + + private VectorIndexOptions() { + } + + /** + * Validates the complete option map and returns canonical values for persistence. + * + * <p>The returned map contains every supported option, including explicit defaults. Unknown, + * retired, misspelled, and invalid options are rejected instead of being silently ignored. + */ + public static Map<String, String> validateAndNormalize(Map<String, String> options) { + Set<String> unknownOptions = new HashSet<>(options.keySet()); + unknownOptions.removeAll(SUPPORTED_OPTIONS); + if (!unknownOptions.isEmpty()) { + throw new IllegalArgumentException("Unsupported vector index options: " + unknownOptions); + } + + VectorDistanceMetric metric = getMetric(options); + VectorQuantizer quantizer = getQuantizer(options); + int numClusters = getNumClusters(options); + int maxIter = getMaxIter(options); + int bits = getRaBitQBits(options); + long seed = getRaBitQSeed(options); + boolean assumeNormalized = shouldAssumeNormalizedVectors(options); + int numProbes = getNumProbes(options); + int refineFactor = getRefineFactor(options); + VectorQueryMode queryMode = getQueryMode(options); + VectorStalePolicy stalePolicy = getStalePolicy(options); + + if (numProbes > numClusters) { Review Comment: Resolved in 809d8c44cc: when `vector.query.nprobes` is omitted, it now resolves to `min(DEFAULT_NUM_PROBES, num_clusters)` and persists that value. Explicit `nprobes` values are unchanged and still rejected when they exceed `num_clusters`. ########## hudi-common/src/test/java/org/apache/hudi/common/index/vector/TestVectorIndexOptions.java: ########## @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.common.index.vector; + +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class TestVectorIndexOptions { + + @Test + void testDefaultsAreCanonicalAndComplete() { + assertEquals( + opts( + VectorIndexOptions.METRIC, "cosine", + VectorIndexOptions.QUANTIZER, "IVF_RABITQ", + VectorIndexOptions.NUM_CLUSTERS, "256", + VectorIndexOptions.MAX_ITER, "20", + VectorIndexOptions.RABITQ_BITS, "4", + VectorIndexOptions.RABITQ_SEED, "42", + VectorIndexOptions.RABITQ_ASSUME_NORMALIZED, "false", + VectorIndexOptions.QUERY_NUM_PROBES, "32", + VectorIndexOptions.QUERY_REFINE_FACTOR, "50", + VectorIndexOptions.QUERY_MODE, "exact_rerank", + VectorIndexOptions.QUERY_STALE_POLICY, "fail"), + VectorIndexOptions.validateAndNormalize(opts())); + } + + @Test + void testValuesAreNormalizedForPersistence() { + Map<String, String> normalized = VectorIndexOptions.validateAndNormalize(opts( + VectorIndexOptions.METRIC, "DOT-PRODUCT", + VectorIndexOptions.QUANTIZER, "ivf-rabitq", + VectorIndexOptions.RABITQ_ASSUME_NORMALIZED, "TRUE", + VectorIndexOptions.QUERY_MODE, "EXACT-RERANK", + VectorIndexOptions.QUERY_STALE_POLICY, "WARN")); + + assertEquals("dot_product", normalized.get(VectorIndexOptions.METRIC)); + assertEquals("IVF_RABITQ", normalized.get(VectorIndexOptions.QUANTIZER)); + assertEquals("true", normalized.get(VectorIndexOptions.RABITQ_ASSUME_NORMALIZED)); + assertEquals("exact_rerank", normalized.get(VectorIndexOptions.QUERY_MODE)); + assertEquals("warn", normalized.get(VectorIndexOptions.QUERY_STALE_POLICY)); + assertThrows( + UnsupportedOperationException.class, + () -> normalized.put(VectorIndexOptions.METRIC, "l2")); + } + + @Test + void testEveryMetricQueryModeAndStalePolicyIsAccepted() { + assertCanonical(VectorIndexOptions.METRIC, "cosine", "cosine"); + assertCanonical(VectorIndexOptions.METRIC, "l2", "l2"); + assertCanonical(VectorIndexOptions.METRIC, "dot_product", "dot_product"); + assertCanonical(VectorIndexOptions.QUERY_MODE, "approximate", "approximate"); + assertCanonical(VectorIndexOptions.QUERY_MODE, "exact_rerank", "exact_rerank"); + assertCanonical(VectorIndexOptions.QUERY_STALE_POLICY, "fail", "fail"); + assertCanonical(VectorIndexOptions.QUERY_STALE_POLICY, "warn", "warn"); + assertCanonical(VectorIndexOptions.QUERY_STALE_POLICY, "fallback", "fallback"); + } + + @Test + void testUnknownRetiredAndMisspelledOptionsAreRejected() { + assertInvalidOption("vector.dimension", "128"); + assertInvalidOption("vector.query.nprobe", "8"); + assertInvalidOption("vector.unknown", "value"); + } + + @Test + void testUnsupportedEnumValuesAreRejectedWithOptionContext() { + assertInvalidValueContainsKey(VectorIndexOptions.METRIC, "manhattan"); + assertInvalidValueContainsKey(VectorIndexOptions.QUANTIZER, "pq"); + assertInvalidValueContainsKey(VectorIndexOptions.QUERY_MODE, "fast-ish"); + assertInvalidValueContainsKey(VectorIndexOptions.QUERY_STALE_POLICY, "ignore"); + assertInvalidValueContainsKey(VectorIndexOptions.RABITQ_ASSUME_NORMALIZED, "yes"); + } + + @Test + void testNumericOptionsAreValidatedWithOptionContext() { + assertCanonical(VectorIndexOptions.RABITQ_BITS, "1", "1"); + assertCanonical(VectorIndexOptions.RABITQ_BITS, "8", "8"); + assertInvalidValueContainsKey(VectorIndexOptions.NUM_CLUSTERS, "0"); + assertInvalidValueContainsKey(VectorIndexOptions.MAX_ITER, "-1"); + assertInvalidValueContainsKey(VectorIndexOptions.QUERY_NUM_PROBES, "0"); + assertInvalidValueContainsKey(VectorIndexOptions.QUERY_REFINE_FACTOR, "-1"); + assertInvalidValueContainsKey(VectorIndexOptions.RABITQ_BITS, "0"); + assertInvalidValueContainsKey(VectorIndexOptions.RABITQ_BITS, "9"); + assertInvalidValueContainsKey(VectorIndexOptions.RABITQ_SEED, "many"); + } + + @Test + void testNumProbesMustNotExceedNumClusters() { Review Comment: Resolved in 809d8c44cc: added tests for omitted `nprobes` with cluster counts 1, 4, 31, 32 and 64, plus explicit in-range and out-of-range probe values. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
