hudi-agent commented on code in PR #19310: URL: https://github.com/apache/hudi/pull/19310#discussion_r4149427999
########## hudi-common/src/main/java/org/apache/hudi/common/index/vector/VectorDistanceMetric.java: ########## @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.common.index.vector; + +import java.util.Locale; + +/** + * Distance metrics for vector similarity search. + * + * <p>All metrics are returned as distances (smaller = more similar), + * so they can be compared uniformly with a min-heap. + */ +public enum VectorDistanceMetric { + + /** + * Cosine distance: 1 - cosine_similarity. + * Range: [0, 2]. 0 = identical direction, 2 = opposite. + */ + COSINE { + @Override + public float compute(float[] a, float[] b) { + checkDimensions(a, b); + double dot = 0; + double normA = 0; + double normB = 0; + for (int i = 0; i < a.length; i++) { + dot += (double) a[i] * b[i]; + normA += (double) a[i] * a[i]; + normB += (double) b[i] * b[i]; + } + double denom = Math.sqrt(normA) * Math.sqrt(normB); + return denom == 0.0 ? 1.0f : (float) (1.0 - dot / denom); + } + }, + + /** + * Euclidean (L2) distance. + * Range: [0, ∞). 0 = identical. + */ + L2 { + @Override + public float compute(float[] a, float[] b) { + checkDimensions(a, b); + double sum = 0; + for (int i = 0; i < a.length; i++) { + double d = (double) a[i] - b[i]; + sum += d * d; + } + return (float) Math.sqrt(sum); + } + }, + + /** + * Maximum inner product distance: negated dot product. + * Negated so smaller = higher similarity, consistent with the min-heap contract. + */ + DOT_PRODUCT { + @Override + public float compute(float[] a, float[] b) { + checkDimensions(a, b); + double dot = 0; + for (int i = 0; i < a.length; i++) { + dot += (double) a[i] * b[i]; + } + return (float) -dot; + } + }; + + /** + * Compute the distance between two float vectors. + * + * @param a first vector + * @param b second vector + * @return non-negative distance (smaller = more similar) + */ + public abstract float compute(float[] a, float[] b); Review Comment: 🤖 nit: the `@return` doc says "non-negative distance", but `DOT_PRODUCT` returns the negated dot product, which can be negative. Could you reword it to something like "distance where smaller = more similar (may be negative for DOT_PRODUCT)"? <sub><i>⚠️ AI-generated; verify before applying. React 👍/👎 to flag quality.</i></sub> ########## hudi-common/src/main/java/org/apache/hudi/common/index/vector/VectorIndexOptions.java: ########## @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.common.index.vector; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * Options accepted by {@code CREATE INDEX ... USING VECTOR}. + * + * <p>The indexed column's Hudi {@code VECTOR(D[, elementType])} schema is authoritative for + * dimension and element type. Index options configure only the acceleration structure. DDL + * implementations must call {@link #validateAndNormalize(Map)} before persisting an index + * definition; individual parsing helpers are intentionally private so aggregate validation cannot + * be bypassed. + */ +public final class VectorIndexOptions { + + public static final String METRIC = "vector.metric"; + public static final String QUANTIZER = "vector.quantizer"; + public static final String NUM_CLUSTERS = "vector.num_clusters"; + public static final String MAX_ITER = "vector.max_iter"; + public static final String RABITQ_BITS = "vector.rabitq.bits"; + public static final String RABITQ_SEED = "vector.rabitq.seed"; + public static final String RABITQ_ASSUME_NORMALIZED = "vector.rabitq.assume_normalized"; + public static final String QUERY_NUM_PROBES = "vector.query.nprobes"; + public static final String QUERY_REFINE_FACTOR = "vector.query.refine_factor"; + public static final String QUERY_MODE = "vector.query.mode"; + public static final String QUERY_STALE_POLICY = "vector.query.stale_policy"; + + public static final VectorDistanceMetric DEFAULT_METRIC = VectorDistanceMetric.COSINE; + public static final VectorQuantizer DEFAULT_QUANTIZER = VectorQuantizer.IVF_RABITQ; + public static final int DEFAULT_NUM_CLUSTERS = 256; + public static final int DEFAULT_MAX_ITER = 20; + public static final int DEFAULT_RABITQ_BITS = 4; + public static final long DEFAULT_RABITQ_SEED = 42L; + public static final int DEFAULT_NUM_PROBES = 32; + public static final int DEFAULT_REFINE_FACTOR = 50; + public static final VectorQueryMode DEFAULT_QUERY_MODE = VectorQueryMode.EXACT_RERANK; + public static final VectorStalePolicy DEFAULT_STALE_POLICY = VectorStalePolicy.FAIL; + + private static final Set<String> SUPPORTED_OPTIONS = Collections.unmodifiableSet( + new HashSet<>(Arrays.asList( + METRIC, + QUANTIZER, + NUM_CLUSTERS, + MAX_ITER, + RABITQ_BITS, + RABITQ_SEED, + RABITQ_ASSUME_NORMALIZED, + QUERY_NUM_PROBES, + QUERY_REFINE_FACTOR, + QUERY_MODE, + QUERY_STALE_POLICY))); + + private VectorIndexOptions() { + } + + /** + * Validates the complete option map and returns canonical values for persistence. + * + * <p>The returned map contains every supported option, including explicit defaults. Unknown, + * retired, misspelled, and invalid options are rejected instead of being silently ignored. + */ + public static Map<String, String> validateAndNormalize(Map<String, String> options) { + Set<String> unknownOptions = new HashSet<>(options.keySet()); + unknownOptions.removeAll(SUPPORTED_OPTIONS); + if (!unknownOptions.isEmpty()) { + throw new IllegalArgumentException("Unsupported vector index options: " + unknownOptions); + } + + VectorDistanceMetric metric = getMetric(options); + VectorQuantizer quantizer = getQuantizer(options); + int numClusters = getNumClusters(options); + int maxIter = getMaxIter(options); + int bits = getRaBitQBits(options); + long seed = getRaBitQSeed(options); + boolean assumeNormalized = shouldAssumeNormalizedVectors(options); + int numProbes = options.containsKey(QUERY_NUM_PROBES) + ? getNumProbes(options) : Math.min(DEFAULT_NUM_PROBES, numClusters); + int refineFactor = getRefineFactor(options); + VectorQueryMode queryMode = getQueryMode(options); + VectorStalePolicy stalePolicy = getStalePolicy(options); + + if (numProbes > numClusters) { + throw new IllegalArgumentException( + "Option '" + QUERY_NUM_PROBES + "' must not exceed '" + NUM_CLUSTERS + "': " + + numProbes + " > " + numClusters); + } + + Map<String, String> normalized = new LinkedHashMap<>(); + normalized.put(METRIC, metric.name().toLowerCase(Locale.ROOT)); + normalized.put(QUANTIZER, quantizer.name()); + normalized.put(NUM_CLUSTERS, String.valueOf(numClusters)); + normalized.put(MAX_ITER, String.valueOf(maxIter)); + normalized.put(RABITQ_BITS, String.valueOf(bits)); + normalized.put(RABITQ_SEED, String.valueOf(seed)); + normalized.put(RABITQ_ASSUME_NORMALIZED, String.valueOf(assumeNormalized)); + normalized.put(QUERY_NUM_PROBES, String.valueOf(numProbes)); + normalized.put(QUERY_REFINE_FACTOR, String.valueOf(refineFactor)); + normalized.put(QUERY_MODE, queryMode.name().toLowerCase(Locale.ROOT)); + normalized.put(QUERY_STALE_POLICY, stalePolicy.name().toLowerCase(Locale.ROOT)); + return Collections.unmodifiableMap(normalized); + } + + private static VectorDistanceMetric getMetric(Map<String, String> options) { Review Comment: 🤖 nit: `getMetric`, `getQuantizer`, `getQueryMode` and `getStalePolicy` all follow the same get/parse/wrap-exception steps. Have you considered a generic `getEnum(options, key, default, Function<String, E> parser)` helper to cut down the repetition? <sub><i>⚠️ AI-generated; verify before applying. React 👍/👎 to flag quality.</i></sub> ########## hudi-common/src/main/java/org/apache/hudi/common/index/vector/VectorIndexOptions.java: ########## @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.common.index.vector; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * Options accepted by {@code CREATE INDEX ... USING VECTOR}. + * + * <p>The indexed column's Hudi {@code VECTOR(D[, elementType])} schema is authoritative for + * dimension and element type. Index options configure only the acceleration structure. DDL + * implementations must call {@link #validateAndNormalize(Map)} before persisting an index + * definition; individual parsing helpers are intentionally private so aggregate validation cannot + * be bypassed. + */ +public final class VectorIndexOptions { + + public static final String METRIC = "vector.metric"; + public static final String QUANTIZER = "vector.quantizer"; + public static final String NUM_CLUSTERS = "vector.num_clusters"; + public static final String MAX_ITER = "vector.max_iter"; + public static final String RABITQ_BITS = "vector.rabitq.bits"; + public static final String RABITQ_SEED = "vector.rabitq.seed"; + public static final String RABITQ_ASSUME_NORMALIZED = "vector.rabitq.assume_normalized"; + public static final String QUERY_NUM_PROBES = "vector.query.nprobes"; + public static final String QUERY_REFINE_FACTOR = "vector.query.refine_factor"; + public static final String QUERY_MODE = "vector.query.mode"; + public static final String QUERY_STALE_POLICY = "vector.query.stale_policy"; + + public static final VectorDistanceMetric DEFAULT_METRIC = VectorDistanceMetric.COSINE; + public static final VectorQuantizer DEFAULT_QUANTIZER = VectorQuantizer.IVF_RABITQ; + public static final int DEFAULT_NUM_CLUSTERS = 256; + public static final int DEFAULT_MAX_ITER = 20; + public static final int DEFAULT_RABITQ_BITS = 4; + public static final long DEFAULT_RABITQ_SEED = 42L; + public static final int DEFAULT_NUM_PROBES = 32; + public static final int DEFAULT_REFINE_FACTOR = 50; + public static final VectorQueryMode DEFAULT_QUERY_MODE = VectorQueryMode.EXACT_RERANK; + public static final VectorStalePolicy DEFAULT_STALE_POLICY = VectorStalePolicy.FAIL; + + private static final Set<String> SUPPORTED_OPTIONS = Collections.unmodifiableSet( + new HashSet<>(Arrays.asList( + METRIC, + QUANTIZER, + NUM_CLUSTERS, + MAX_ITER, + RABITQ_BITS, + RABITQ_SEED, + RABITQ_ASSUME_NORMALIZED, + QUERY_NUM_PROBES, + QUERY_REFINE_FACTOR, + QUERY_MODE, + QUERY_STALE_POLICY))); + + private VectorIndexOptions() { + } + + /** + * Validates the complete option map and returns canonical values for persistence. + * + * <p>The returned map contains every supported option, including explicit defaults. Unknown, + * retired, misspelled, and invalid options are rejected instead of being silently ignored. + */ + public static Map<String, String> validateAndNormalize(Map<String, String> options) { + Set<String> unknownOptions = new HashSet<>(options.keySet()); + unknownOptions.removeAll(SUPPORTED_OPTIONS); + if (!unknownOptions.isEmpty()) { + throw new IllegalArgumentException("Unsupported vector index options: " + unknownOptions); + } + + VectorDistanceMetric metric = getMetric(options); + VectorQuantizer quantizer = getQuantizer(options); + int numClusters = getNumClusters(options); + int maxIter = getMaxIter(options); + int bits = getRaBitQBits(options); + long seed = getRaBitQSeed(options); + boolean assumeNormalized = shouldAssumeNormalizedVectors(options); + int numProbes = options.containsKey(QUERY_NUM_PROBES) + ? getNumProbes(options) : Math.min(DEFAULT_NUM_PROBES, numClusters); + int refineFactor = getRefineFactor(options); + VectorQueryMode queryMode = getQueryMode(options); + VectorStalePolicy stalePolicy = getStalePolicy(options); + + if (numProbes > numClusters) { + throw new IllegalArgumentException( + "Option '" + QUERY_NUM_PROBES + "' must not exceed '" + NUM_CLUSTERS + "': " + + numProbes + " > " + numClusters); + } + + Map<String, String> normalized = new LinkedHashMap<>(); + normalized.put(METRIC, metric.name().toLowerCase(Locale.ROOT)); Review Comment: 🤖 nit: the quantizer is persisted in upper case (`IVF_RABITQ`), while metric, query mode and stale policy are lower-cased. Could we lower-case it too, so the persisted canonical form is consistent? <sub><i>⚠️ AI-generated; verify before applying. React 👍/👎 to flag quality.</i></sub> -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
