[
https://issues.apache.org/jira/browse/KUDU-3414?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Bakai Ádám updated KUDU-3414:
-----------------------------
Description:
Reproduction:
1. Start new master and tablet servers.
2. Create a client and create a table and do a succesfull write operation with
it.
3. Pause the leader tablet server.
4. Now if you attempt to write with the same client, it will timeout, and it
will try negotiation with the same server until the session times out. (If a
new client is created, then the write operation is succesful.)
KUDU-1580 works on a similar issue, the error is retriable for now, but
[https://gerrit.cloudera.org/#/c/6926/] states that "the Kudu C++ client
retries an RPC with other tablet replica if the connection negotiation with
current replica timed out." which doesn't happen right now. (Didn't check if
the current behaviour is a regression or not.)
Here is a reproduction test code into client-test.cc:
{code:java}
TEST_F(ClientTest, TestDefaultRPCTimeoutSessionTimeoutDifferent) {
cluster_->Shutdown();
FLAGS_raft_heartbeat_interval_ms = 50;
FLAGS_leader_failure_exp_backoff_max_delta_ms = 1000;
const string table_name = "TestTimeout";
unique_ptr<ExternalMiniCluster> ext_cluster_;
ext_cluster_.reset(new ExternalMiniCluster( ExternalMiniClusterOptions()));
ext_cluster_->Start();
ext_cluster_->AddMaster();
ext_cluster_->AddTabletServer();
ext_cluster_->AddTabletServer();
ext_cluster_->AddTabletServer();
ext_cluster_->AddTabletServer();
ext_cluster_->AddTabletServer();
ext_cluster_->WaitForTabletServerCount(5,MonoDelta::FromSeconds(1));
client::sp::shared_ptr<client::KuduClient> client;
client_= nullptr;
KuduClientBuilder builder;
builder.num_reactors(1);
builder.connection_negotiation_timeout(MonoDelta::FromMilliseconds(1500));
builder.default_rpc_timeout(MonoDelta::FromMilliseconds(500));
ext_cluster_->CreateClient(&builder,&client);
unique_ptr<KuduTableCreator> table_creator(client->NewTableCreator());
table_creator->table_name(table_name)
.schema(&schema_)
.num_replicas(5)
.set_range_partition_columns({ "key" })
.timeout(MonoDelta::FromSeconds(60))
.Create();
shared_ptr<KuduTable> table;
client->OpenTable(table_name,&table);
unique_ptr<KuduInsert> insert;
shared_ptr<KuduSession> session = client->NewSession();
session->SetTimeoutMillis(60000);
insert = BuildTestInsert(table.get(), 1);
ASSERT_OK(session->Apply(insert.release()));
session->Flush();
KuduScanTokenBuilder builder2(table.get());
vector<KuduScanToken*> tokens;
ASSERT_OK(builder2.Build(&tokens));
const KuduTablet& tablet = tokens[0]->tablet();
for (const auto& a: tablet.replicas())
{
if (a->is_leader())
{
ext_cluster_->tablet_server_by_uuid(a->ts().uuid())->Pause();
//ext_cluster_->tablet_server_by_uuid(a->ts().uuid())->Shutdown();
break;
}
}
LOG(INFO)<<"paused";
//ext_cluster_->CreateClient(&builder,&client);
session = client->NewSession();
session->SetTimeoutMillis(60000);
insert = BuildTestInsert(table.get(), 2);
ASSERT_OK(session->Apply(insert.release()));
}
{code}
(Don't forget to add the declarations and includes to the start of the
[file:|file:///])
{code:java}
#include "kudu/mini-cluster/external_mini_cluster.h"
using kudu::cluster::ExternalMiniCluster;
using kudu::cluster::ExternalMiniClusterOptions;
{code}
The logic about deciding when to try different tablet is in:
RetriableRpcStatus WriteRpc::AnalyzeResponse(const Status& rpc_cb_status)
was:
Reproduction:
1. Start new master and tablet servers.
2. Create a client and create a table and do a succesfull write operation with
it.
3. Pause the leader tablet server.
4. Now if you attempt to write with the same client, it will timeout, and it
will try negotiation with the same server until the session times out. (If a
new client is created, then the write operation is succesful.)
KUDU-1580 works on a similar issue, the error is retriable for now, but
[https://gerrit.cloudera.org/#/c/6926/] states that "the Kudu C++ client
retries an RPC with other tablet replica if the connection negotiation with
current replica timed out." which doesn't happen right now. (Didn't check if
the current behaviour is a regression or not.)
Here is a reproduction test code into client-test.cc:
{code:java}
TEST_F(ClientTest, TestDefaultRPCTimeoutSessionTimeoutDifferent) {
cluster_->Shutdown();
FLAGS_raft_heartbeat_interval_ms = 50;
FLAGS_leader_failure_exp_backoff_max_delta_ms = 1000;
const string table_name = "TestTimeout";
unique_ptr<ExternalMiniCluster> ext_cluster_;
ext_cluster_.reset(new ExternalMiniCluster( ExternalMiniClusterOptions()));
ext_cluster_->Start();
ext_cluster_->AddMaster();
ext_cluster_->AddTabletServer();
ext_cluster_->AddTabletServer();
ext_cluster_->AddTabletServer();
ext_cluster_->AddTabletServer();
ext_cluster_->AddTabletServer();
ext_cluster_->WaitForTabletServerCount(5,MonoDelta::FromSeconds(1));
client::sp::shared_ptr<client::KuduClient> client;
client_= nullptr;
KuduClientBuilder builder;
builder.num_reactors(1);
builder.connection_negotiation_timeout(MonoDelta::FromMilliseconds(1500));
builder.default_rpc_timeout(MonoDelta::FromMilliseconds(500));
ext_cluster_->CreateClient(&builder,&client);
unique_ptr<KuduTableCreator> table_creator(client->NewTableCreator());
table_creator->table_name(table_name)
.schema(&schema_)
.num_replicas(5)
.set_range_partition_columns({ "key" })
.timeout(MonoDelta::FromSeconds(60))
.Create();
shared_ptr<KuduTable> table;
client->OpenTable(table_name,&table);
unique_ptr<KuduInsert> insert;
shared_ptr<KuduSession> session = client->NewSession();
session->SetTimeoutMillis(60000);
insert = BuildTestInsert(table.get(), 1);
ASSERT_OK(session->Apply(insert.release()));
session->Flush();
KuduScanTokenBuilder builder2(table.get());
vector<KuduScanToken*> tokens;
ASSERT_OK(builder2.Build(&tokens));
const KuduTablet& tablet = tokens[0]->tablet();
for (const auto& a: tablet.replicas())
{
if (a->is_leader())
{
ext_cluster_->tablet_server_by_uuid(a->ts().uuid())->Pause();
//ext_cluster_->tablet_server_by_uuid(a->ts().uuid())->Shutdown();
break;
}
}
LOG(INFO)<<"paused";
//ext_cluster_->CreateClient(&builder,&client);
session = client->NewSession();
session->SetTimeoutMillis(60000);
insert = BuildTestInsert(table.get(), 2);
ASSERT_OK(session->Apply(insert.release()));
}
{code}
(Don't forget to add the declarations and includes to the start of the file:)
{code:java}
#include "kudu/mini-cluster/external_mini_cluster.h"
using kudu::cluster::ExternalMiniCluster;
using kudu::cluster::ExternalMiniClusterOptions;
{code}
> Client tries with the same address over and over in case of negotiation
> timeout
> -------------------------------------------------------------------------------
>
> Key: KUDU-3414
> URL: https://issues.apache.org/jira/browse/KUDU-3414
> Project: Kudu
> Issue Type: Bug
> Components: client
> Reporter: Bakai Ádám
> Priority: Major
>
> Reproduction:
> 1. Start new master and tablet servers.
> 2. Create a client and create a table and do a succesfull write operation
> with it.
> 3. Pause the leader tablet server.
> 4. Now if you attempt to write with the same client, it will timeout, and it
> will try negotiation with the same server until the session times out. (If a
> new client is created, then the write operation is succesful.)
> KUDU-1580 works on a similar issue, the error is retriable for now, but
> [https://gerrit.cloudera.org/#/c/6926/] states that "the Kudu C++ client
> retries an RPC with other tablet replica if the connection negotiation with
> current replica timed out." which doesn't happen right now. (Didn't check if
> the current behaviour is a regression or not.)
> Here is a reproduction test code into client-test.cc:
> {code:java}
> TEST_F(ClientTest, TestDefaultRPCTimeoutSessionTimeoutDifferent) {
> cluster_->Shutdown();
> FLAGS_raft_heartbeat_interval_ms = 50;
> FLAGS_leader_failure_exp_backoff_max_delta_ms = 1000;
> const string table_name = "TestTimeout";
> unique_ptr<ExternalMiniCluster> ext_cluster_;
> ext_cluster_.reset(new ExternalMiniCluster(
> ExternalMiniClusterOptions()));
> ext_cluster_->Start();
> ext_cluster_->AddMaster();
> ext_cluster_->AddTabletServer();
> ext_cluster_->AddTabletServer();
> ext_cluster_->AddTabletServer();
> ext_cluster_->AddTabletServer();
> ext_cluster_->AddTabletServer();
> ext_cluster_->WaitForTabletServerCount(5,MonoDelta::FromSeconds(1));
> client::sp::shared_ptr<client::KuduClient> client;
> client_= nullptr;
> KuduClientBuilder builder;
> builder.num_reactors(1);
> builder.connection_negotiation_timeout(MonoDelta::FromMilliseconds(1500));
> builder.default_rpc_timeout(MonoDelta::FromMilliseconds(500));
> ext_cluster_->CreateClient(&builder,&client);
> unique_ptr<KuduTableCreator> table_creator(client->NewTableCreator());
> table_creator->table_name(table_name)
> .schema(&schema_)
> .num_replicas(5)
> .set_range_partition_columns({ "key" })
> .timeout(MonoDelta::FromSeconds(60))
> .Create();
> shared_ptr<KuduTable> table;
> client->OpenTable(table_name,&table);
> unique_ptr<KuduInsert> insert;
> shared_ptr<KuduSession> session = client->NewSession();
> session->SetTimeoutMillis(60000);
> insert = BuildTestInsert(table.get(), 1);
> ASSERT_OK(session->Apply(insert.release()));
> session->Flush();
> KuduScanTokenBuilder builder2(table.get());
> vector<KuduScanToken*> tokens;
> ASSERT_OK(builder2.Build(&tokens));
> const KuduTablet& tablet = tokens[0]->tablet();
> for (const auto& a: tablet.replicas())
> {
> if (a->is_leader())
> {
> ext_cluster_->tablet_server_by_uuid(a->ts().uuid())->Pause();
> //ext_cluster_->tablet_server_by_uuid(a->ts().uuid())->Shutdown();
> break;
> }
> }
> LOG(INFO)<<"paused";
> //ext_cluster_->CreateClient(&builder,&client);
> session = client->NewSession();
> session->SetTimeoutMillis(60000);
> insert = BuildTestInsert(table.get(), 2);
> ASSERT_OK(session->Apply(insert.release()));
> }
> {code}
> (Don't forget to add the declarations and includes to the start of the
> [file:|file:///])
> {code:java}
> #include "kudu/mini-cluster/external_mini_cluster.h"
> using kudu::cluster::ExternalMiniCluster;
> using kudu::cluster::ExternalMiniClusterOptions;
> {code}
> The logic about deciding when to try different tablet is in:
> RetriableRpcStatus WriteRpc::AnalyzeResponse(const Status& rpc_cb_status)
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
