u70b3 commented on code in PR #67754: URL: https://github.com/apache/doris/pull/67754#discussion_r4141832502
########## fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ResolveLanceIndexJobCommand.java: ########## @@ -0,0 +1,367 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.CatalogMgr; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.lance.LanceExternalCatalog; +import org.apache.doris.datasource.lance.LanceIndexMutationValidator; +import org.apache.doris.datasource.lance.job.LanceIndexJob; +import org.apache.doris.datasource.lance.job.LanceIndexJobManager; +import org.apache.doris.datasource.lance.job.LanceIndexJobMutationState; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.StmtExecutor; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.nio.charset.StandardCharsets; + +/** + * RESOLVE LANCE INDEX JOB <jobId> AS FORCE_RELEASE COMMENT '<note>' — the operator + * escape hatch that durably releases a job whose mutation outcome is UNKNOWN (design section + * 7.1). RESOLVE is deliberately not gated by {@code enable_lance_index_mutation}: the gate + * controls mutation admission, while FORCE must stay available exactly when the gate is off. + * + * <p>The release protocol keeps the fence, the quota charge and the possible-live slot while + * it performs one authoritative latest-metadata read and one external-table refresh with the + * current credentials of the surviving catalog, both outside every catalog/manager lock; only + * then does the durable release transfer inside the admission critical section + * ({@code captureLanceIndexTarget} → lock-free read/refresh → {@code withLanceIndexAdmission} + * recheck → manager write lock), serialized against DROP CATALOG and identity ALTER exactly + * like admission. Any failure before the transfer is the typed + * {@code ERR_LANCE_INDEX_JOB_RESOLUTION_INCOMPLETE}: nothing is written, nothing is released, + * and the operator fixes the cause and retries the same statement. + * + * <p>Target resolution (design section 7.1 step 1) is three-valued. RESOLVED means the + * persisted names resolve and the catalog's current durable dataset locator still matches + * the job's — the same revalidation SHOW LANCE INDEX JOBS applies, so a repointed dataset + * reusing the same names never turns a stale name into table-level authorization. MISSING + * means the catalog, database or table is verifiably absent, or the locator positively + * points at a different dataset: that is the orphan family — a full orphan (catalog gone) + * has no credentials to read with and nothing to invalidate, so it is released directly + * after global ADMIN authorization, while a half-orphan skips the authoritative read and + * refreshes with {@code ignoreIfNotExists=true} as a best-effort invalidation. FAILED means + * a resolution that errors out, or a locator that cannot be resolved right now: never an + * orphan verdict — after ADMIN authorization the statement fails with the typed 5105 so the + * fence is kept when "table gone" cannot be told apart from "network down". SHOW fails the + * same uncertainty closed by hiding the row; RESOLVE fails it closed by not releasing. + * + * <p>Non-disclosure (design section 8): the job is loaded first and authorized against its + * persisted target — table-level ALTER when the target resolves, global ADMIN otherwise — and + * a missing job and an unauthorized job share the same fixed ERR_LANCE_INDEX_JOB_NOT_FOUND + * response naming only the job id. The 5104 state rejection and the 5105 resolution failure + * are only visible to an already authorized caller. + * + * <p>Success returns an OK packet carrying one warning row with {@link #LATE_COMMIT_WARNING}, + * the same text persisted as the job's durable {@code forceWarning}: the old worker may still + * overwrite, remove, or reintroduce the index name; the mutation outcome remains UNKNOWN. + * Retrying FORCE on an already released job is an idempotent success returning the existing + * release record, never an error. + */ +public class ResolveLanceIndexJobCommand extends Command implements ForwardWithSync { + /** + * The late-commit warning (design section 7.1), returned in the OK packet and persisted + * verbatim as the durable {@code forceWarning}; bounded well under + * {@link LanceIndexJob#MAX_FORCE_TEXT_BYTES}. + */ + static final String LATE_COMMIT_WARNING = + "the old worker may still overwrite, remove, or reintroduce the index name; " + + "the mutation outcome remains UNKNOWN"; + + private static final Logger LOG = LogManager.getLogger(ResolveLanceIndexJobCommand.class); + + private final long jobId; + private final String comment; + + public ResolveLanceIndexJobCommand(long jobId, String comment) { + super(PlanType.RESOLVE_LANCE_INDEX_JOB_COMMAND); + this.jobId = jobId; + this.comment = comment; + } + + public long getJobId() { + return jobId; + } + + public String getComment() { + return comment; + } + + @Override + public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { + Env env = Env.getCurrentEnv(); + LanceIndexJobManager manager = env.getLanceIndexJobManager(); + // 1. Load the job without disclosing any field (design section 7.1 step 1). + LanceIndexJob job = manager.getJob(jobId); + if (job == null) { + throw notFound(); + } + // 2. Resolve and authorize against the persisted target before any state is revealed: + // table-level ALTER when the target resolves, global ADMIN for the orphan family + // and for a target whose resolution failed outright. + CatalogMgr catalogMgr = env.getCatalogMgr(); + CatalogIf<? extends DatabaseIf<? extends TableIf>> catalog = catalogMgr.getCatalog(job.getCatalogId()); + TargetResolution resolution = resolveTarget(catalog, job); + boolean authorized = resolution == TargetResolution.RESOLVED + ? env.getAccessManager().checkTblPriv(ctx, catalog.getName(), job.getDbName(), job.getTableName(), + PrivPredicate.ALTER) + : env.getAccessManager().checkGlobalPriv(ctx, PrivPredicate.ADMIN); + if (!authorized) { + throw notFound(); + } + // 3. Idempotent replay: a retry returns the existing release record (section 7.1). + // This deliberately precedes the resolution-failure rejection: once the release + // has landed, a retry during a provider outage is a success, not a 5105. + if (job.isForceReleased()) { Review Comment: done: the idempotent shortcut now requires forceReleased AND a null/UNKNOWN mutation state, mirroring the manager gate; a malformed forceReleased PENDING/RUNNING record falls through to the 5104 state rejection instead of a false OK (testForceReleasedNonUnknownRecordIsNotIdempotentSuccess, 59d120c268). -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
