tombentley commented on a change in pull request #11777: URL: https://github.com/apache/kafka/pull/11777#discussion_r816647178
########## File path: clients/src/main/java/org/apache/kafka/clients/admin/internals/FenceProducersHandler.java ########## @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.admin.internals; + +import org.apache.kafka.common.Node; +import org.apache.kafka.common.errors.ClusterAuthorizationException; +import org.apache.kafka.common.errors.TransactionalIdAuthorizationException; +import org.apache.kafka.common.message.InitProducerIdRequestData; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.FindCoordinatorRequest; +import org.apache.kafka.common.requests.InitProducerIdRequest; +import org.apache.kafka.common.requests.InitProducerIdResponse; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.ProducerIdAndEpoch; +import org.slf4j.Logger; + +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class FenceProducersHandler extends AdminApiHandler.Unbatched<CoordinatorKey, ProducerIdAndEpoch> { + private final Logger log; + private final AdminApiLookupStrategy<CoordinatorKey> lookupStrategy; + + public FenceProducersHandler( + LogContext logContext + ) { + this.log = logContext.logger(FenceProducersHandler.class); + this.lookupStrategy = new CoordinatorStrategy(FindCoordinatorRequest.CoordinatorType.TRANSACTION, logContext); + } + + public static AdminApiFuture.SimpleAdminApiFuture<CoordinatorKey, ProducerIdAndEpoch> newFuture( + Collection<String> transactionalIds + ) { + return AdminApiFuture.forKeys(buildKeySet(transactionalIds)); + } + + private static Set<CoordinatorKey> buildKeySet(Collection<String> transactionalIds) { + return transactionalIds.stream() + .map(CoordinatorKey::byTransactionalId) + .collect(Collectors.toSet()); + } + + @Override + public String apiName() { + return "fenceProducer"; + } + + @Override + public AdminApiLookupStrategy<CoordinatorKey> lookupStrategy() { + return lookupStrategy; + } + + @Override + InitProducerIdRequest.Builder buildSingleRequest(int brokerId, CoordinatorKey key) { + if (key.type != FindCoordinatorRequest.CoordinatorType.TRANSACTION) { + throw new IllegalArgumentException("Invalid group coordinator key " + key + + " when building `InitProducerId` request"); + } + InitProducerIdRequestData data = new InitProducerIdRequestData() + // Because we never include a producer epoch or ID in this request, we expect that some errors + // (such as PRODUCER_FENCED) will never be returned in the corresponding broker response. + // If we ever modify this logic to include an epoch or producer ID, we will need to update the + // error handling logic for this handler to accommodate these new errors. + .setProducerEpoch(ProducerIdAndEpoch.NONE.epoch) + .setProducerId(ProducerIdAndEpoch.NONE.producerId) + .setTransactionalId(key.idValue) + // Set transaction timeout to 1 since it's only being initialized to fence out older producers with the same transactional ID, + // and shouldn't be used for any actual record writes + .setTransactionTimeoutMs(1); + return new InitProducerIdRequest.Builder(data); + } + + @Override + public ApiResult<CoordinatorKey, ProducerIdAndEpoch> handleSingleResponse( + Node broker, + CoordinatorKey key, + AbstractResponse abstractResponse + ) { + InitProducerIdResponse response = (InitProducerIdResponse) abstractResponse; + + Errors error = Errors.forCode(response.data().errorCode()); + if (error != Errors.NONE) { + return handleError(key, error); + } + + Map<CoordinatorKey, ProducerIdAndEpoch> completed = Collections.singletonMap(key, new ProducerIdAndEpoch( + response.data().producerId(), + response.data().producerEpoch() + )); + + return new ApiResult<>(completed, Collections.emptyMap(), Collections.emptyList()); + } + + private ApiResult<CoordinatorKey, ProducerIdAndEpoch> handleError(CoordinatorKey transactionalIdKey, Errors error) { + switch (error) { + case CLUSTER_AUTHORIZATION_FAILED: + return ApiResult.failed(transactionalIdKey, new ClusterAuthorizationException( + "InitProducerId request for transactionalId `" + transactionalIdKey.idValue + "` " + + "failed due to cluster authorization failure")); + + case TRANSACTIONAL_ID_AUTHORIZATION_FAILED: + return ApiResult.failed(transactionalIdKey, new TransactionalIdAuthorizationException( + "InitProducerId request for transactionalId `" + transactionalIdKey.idValue + "` " + + "failed due to transactional ID authorization failure")); + + // We intentionally omit cases for PRODUCER_FENCED, TRANSACTIONAL_ID_NOT_FOUND, and INVALID_PRODUCER_EPOCH + // since those errors should never happen when our InitProducerIdRequest doesn't include a producer epoch or ID + // and should therefore fall under the "unexpected error" catch-all case below + + case COORDINATOR_LOAD_IN_PROGRESS: + // If the coordinator is in the middle of loading, then we just need to retry + log.debug("InitProducerId request for transactionalId `{}` failed because the " + + "coordinator is still in the process of loading state. Will retry", + transactionalIdKey.idValue); + return ApiResult.empty(); + + case NOT_COORDINATOR: + case COORDINATOR_NOT_AVAILABLE: + // If the coordinator is unavailable or there was a coordinator change, then we unmap + // the key so that we retry the `FindCoordinator` request + log.debug("InitProducerId request for transactionalId `{}` returned error {}. Will attempt " + + "to find the coordinator again and retry", transactionalIdKey.idValue, error); Review comment: "find the coordinator again and retry" sounds a bit weird. Isn't it enough to say "retry"? ########## File path: clients/src/main/java/org/apache/kafka/clients/admin/FenceProducersResult.java ########## @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import org.apache.kafka.clients.admin.internals.CoordinatorKey; +import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.annotation.InterfaceStability; +import org.apache.kafka.common.utils.ProducerIdAndEpoch; + +import java.util.Collection; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * The result of the {@link Admin#fenceProducers(Collection)} call. + * + * The API of this class is evolving, see {@link Admin} for details. + */ +@InterfaceStability.Evolving +public class FenceProducersResult { + + private final Map<CoordinatorKey, KafkaFuture<ProducerIdAndEpoch>> futures; + + FenceProducersResult(Map<CoordinatorKey, KafkaFuture<ProducerIdAndEpoch>> futures) { + this.futures = futures; + } + + /** + * Return a map from transactional ID to futures which can be used to check the status of + * individual fencings. + */ + public Map<String, KafkaFuture<Void>> fencedProducers() { Review comment: It feels like a bit of a shame that we didn't propose to a public version of `ProducerIdAndEpoch`, which would result in a more natural API than this `Void` future plus the methods for getting the PID and epoch for individual transactional ids. ########## File path: clients/src/main/java/org/apache/kafka/clients/admin/internals/FenceProducersHandler.java ########## @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.admin.internals; + +import org.apache.kafka.common.Node; +import org.apache.kafka.common.errors.ClusterAuthorizationException; +import org.apache.kafka.common.errors.TransactionalIdAuthorizationException; +import org.apache.kafka.common.message.InitProducerIdRequestData; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.FindCoordinatorRequest; +import org.apache.kafka.common.requests.InitProducerIdRequest; +import org.apache.kafka.common.requests.InitProducerIdResponse; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.ProducerIdAndEpoch; +import org.slf4j.Logger; + +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class FenceProducersHandler extends AdminApiHandler.Unbatched<CoordinatorKey, ProducerIdAndEpoch> { + private final Logger log; + private final AdminApiLookupStrategy<CoordinatorKey> lookupStrategy; + + public FenceProducersHandler( + LogContext logContext + ) { + this.log = logContext.logger(FenceProducersHandler.class); + this.lookupStrategy = new CoordinatorStrategy(FindCoordinatorRequest.CoordinatorType.TRANSACTION, logContext); + } + + public static AdminApiFuture.SimpleAdminApiFuture<CoordinatorKey, ProducerIdAndEpoch> newFuture( + Collection<String> transactionalIds + ) { + return AdminApiFuture.forKeys(buildKeySet(transactionalIds)); + } + + private static Set<CoordinatorKey> buildKeySet(Collection<String> transactionalIds) { + return transactionalIds.stream() + .map(CoordinatorKey::byTransactionalId) + .collect(Collectors.toSet()); + } + + @Override + public String apiName() { + return "fenceProducer"; + } + + @Override + public AdminApiLookupStrategy<CoordinatorKey> lookupStrategy() { + return lookupStrategy; + } + + @Override + InitProducerIdRequest.Builder buildSingleRequest(int brokerId, CoordinatorKey key) { + if (key.type != FindCoordinatorRequest.CoordinatorType.TRANSACTION) { + throw new IllegalArgumentException("Invalid group coordinator key " + key + + " when building `InitProducerId` request"); + } + InitProducerIdRequestData data = new InitProducerIdRequestData() + // Because we never include a producer epoch or ID in this request, we expect that some errors + // (such as PRODUCER_FENCED) will never be returned in the corresponding broker response. + // If we ever modify this logic to include an epoch or producer ID, we will need to update the + // error handling logic for this handler to accommodate these new errors. + .setProducerEpoch(ProducerIdAndEpoch.NONE.epoch) + .setProducerId(ProducerIdAndEpoch.NONE.producerId) + .setTransactionalId(key.idValue) + // Set transaction timeout to 1 since it's only being initialized to fence out older producers with the same transactional ID, + // and shouldn't be used for any actual record writes + .setTransactionTimeoutMs(1); + return new InitProducerIdRequest.Builder(data); + } + + @Override + public ApiResult<CoordinatorKey, ProducerIdAndEpoch> handleSingleResponse( + Node broker, + CoordinatorKey key, + AbstractResponse abstractResponse + ) { + InitProducerIdResponse response = (InitProducerIdResponse) abstractResponse; + + Errors error = Errors.forCode(response.data().errorCode()); + if (error != Errors.NONE) { + return handleError(key, error); + } + + Map<CoordinatorKey, ProducerIdAndEpoch> completed = Collections.singletonMap(key, new ProducerIdAndEpoch( + response.data().producerId(), + response.data().producerEpoch() + )); + + return new ApiResult<>(completed, Collections.emptyMap(), Collections.emptyList()); + } + + private ApiResult<CoordinatorKey, ProducerIdAndEpoch> handleError(CoordinatorKey transactionalIdKey, Errors error) { + switch (error) { + case CLUSTER_AUTHORIZATION_FAILED: + return ApiResult.failed(transactionalIdKey, new ClusterAuthorizationException( + "InitProducerId request for transactionalId `" + transactionalIdKey.idValue + "` " + + "failed due to cluster authorization failure")); + + case TRANSACTIONAL_ID_AUTHORIZATION_FAILED: + return ApiResult.failed(transactionalIdKey, new TransactionalIdAuthorizationException( + "InitProducerId request for transactionalId `" + transactionalIdKey.idValue + "` " + + "failed due to transactional ID authorization failure")); + + // We intentionally omit cases for PRODUCER_FENCED, TRANSACTIONAL_ID_NOT_FOUND, and INVALID_PRODUCER_EPOCH + // since those errors should never happen when our InitProducerIdRequest doesn't include a producer epoch or ID + // and should therefore fall under the "unexpected error" catch-all case below Review comment: Wouldn't this make more sense just before the `default:` clause? ########## File path: clients/src/main/java/org/apache/kafka/common/utils/Utils.java ########## @@ -1415,4 +1415,21 @@ public static boolean isBlank(String str) { return res; } + /** + * Get an array containing all of the {@link Object#toString names} of a given enumerable type. + * @param enumClass the enum class; may not be null + * @return an array with the names of every value for the enum class; never null, but may be empty + * if there are no values defined for the enum + */ + public static String[] enumOptions(Class<? extends Enum<?>> enumClass) { + Objects.requireNonNull(enumClass); + if (!enumClass.isEnum()) { + throw new IllegalArgumentException("Class " + enumClass + " is not an enumerable type"); + } + + return Stream.of(enumClass.getEnumConstants()) + .map(Object::toString) Review comment: Is it correct to use `Object::toString` here, rather than `Enum::name`? If so, maybe the javadoc needs adjusting since it's not guaranteed to return the enum value name. ########## File path: clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java ########## @@ -6281,6 +6284,30 @@ public void testClientSideTimeoutAfterFailureToReceiveResponse() throws Exceptio } } + @Test + public void testFenceProducers() throws Exception { + try (AdminClientUnitTestEnv env = mockClientEnv()) { + String transactionalId = "copyCat"; + Node transactionCoordinator = env.cluster().nodes().iterator().next(); + + env.kafkaClient().prepareResponse(prepareFindCoordinatorResponse(Errors.NONE, transactionalId, transactionCoordinator)); + + InitProducerIdResponseData initProducerIdResponseData = new InitProducerIdResponseData() + .setProducerId(4761) + .setProducerEpoch((short) 489); + env.kafkaClient().prepareResponseFrom( + request -> request instanceof InitProducerIdRequest, + new InitProducerIdResponse(initProducerIdResponseData), + transactionCoordinator + ); + + FenceProducersResult result = env.adminClient().fenceProducers(Collections.singleton(transactionalId)); + assertNull(result.all().get()); + assertEquals(4761, result.producerId(transactionalId).get()); + assertEquals((short) 489, result.epochId(transactionalId).get()); + } + } + Review comment: Can't we test the `NOT_COORDINATOR` and `COORDINATOR_NOT_AVAILABLE` cases here too? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: jira-unsubscr...@kafka.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
