I'm trying to add a replica but it is failing with "No node can satisfy
the rules" and the diagnostics log entry shows violaitions as []
How can I determine what the issue is and correct it?
DIAGNOSTICS AND STACKTRACE FROM solr.log:
2021-11-08 00:43:38.285 ERROR
(OverseerThreadFactory-38-thread-5-processing-n:solr-0001:8983_solr)
[c:weeklies_2021-10-25_7_7_3 s:shard2 ] o.a.s.c.s.c.a.PolicyHelper
errorId : AutoScaling.error.diagnostics.2010951363830029 {
"sortedNodes":[{
"node":"solr-0002:8983_solr",
"isLive":true,
"cores":4.0,
"freedisk":62.7372932434082,
"sysLoadAvg":28.000000000000004,
"totaldisk":1151.609375,
"replicas":{
"alldata":{"shard1":[{
"core_node64":{
"core":"alldata_shard1_replica_n63",
"shard":"shard1",
"collection":"alldata",
"node_name":"solr-0002:8983_solr",
"type":"NRT",
"base_url":"http://solr-0002:8983/solr";,
"state":"active",
"force_set_state":"false",
"INDEX.sizeInGB":174.76101875957102}}]},
"weeklies_2021-10-25_7_7_3":{
"shard2":[],
"shard1":[{
"core_node3":{
"core":"weeklies_2021-10-25_7_7_3_shard1_replica_n1",
"shard":"shard1",
"collection":"weeklies_2021-10-25_7_7_3",
"node_name":"solr-0002:8983_solr",
"type":"NRT",
"leader":"true",
"base_url":"http://solr-0002:8983/solr";,
"state":"active",
"force_set_state":"false",
"INDEX.sizeInGB":806.7034577354789}}]},
"dailies_2021-1":{"shard2":[{
"core_node22":{
"core":"dailies_2021-1_shard2_replica_n21",
"shard":"shard2",
"collection":"dailies_2021-1",
"node_name":"solr-0002:8983_solr",
"type":"NRT",
"leader":"true",
"base_url":"http://solr-0002:8983/solr";,
"state":"active",
"force_set_state":"false",
"INDEX.sizeInGB":61.86608923971653}}]},
"clinicaltrials_20211014":{"shard1":[{
"core_node28":{
"core":"clinicaltrials_20211014_shard1_replica_n27",
"shard":"shard1",
"collection":"clinicaltrials_20211014",
"node_name":"solr-0002:8983_solr",
"type":"NRT",
"base_url":"http://solr-0002:8983/solr";,
"state":"active",
"force_set_state":"false",
"INDEX.sizeInGB":8.829953823238611}}]}}}
,{
"node":"solr-0001:8983_solr",
"isLive":true,
"cores":4.0,
"freedisk":212.70776748657227,
"sysLoadAvg":42.0,
"totaldisk":1151.609375,
"replicas":{
"alldata":{"shard1":[{
"core_node62":{
"core":"alldata_shard1_replica_n61",
"shard":"shard1",
"collection":"alldata",
"node_name":"solr-0001:8983_solr",
"type":"NRT",
"leader":"true",
"base_url":"http://solr-0001:8983/solr";,
"state":"active",
"force_set_state":"false",
"INDEX.sizeInGB":174.71082909032702}}]},
"weeklies_2021-10-25_7_7_3":{
"shard2":[{
"core_node7":{
"core":"weeklies_2021-10-25_7_7_3_shard2_replica_n4",
"shard":"shard2",
"collection":"weeklies_2021-10-25_7_7_3",
"node_name":"solr-0001:8983_solr",
"type":"NRT",
"leader":"true",
"base_url":"http://solr-0001:8983/solr";,
"state":"active",
"force_set_state":"false",
"INDEX.sizeInGB":670.9012377830222}}],
"shard1":[]},
"dailies_2021-1":{"shard2":[{
"core_node42":{
"core":"dailies_2021-1_shard2_replica_n41",
"shard":"shard2",
"collection":"dailies_2021-1",
"node_name":"solr-0001:8983_solr",
"type":"NRT",
"base_url":"http://solr-0001:8983/solr";,
"state":"active",
"force_set_state":"false",
"INDEX.sizeInGB":61.86608923971653}}]},
"clinicaltrials_20211014":{"shard1":[{
"core_node36":{
"core":"clinicaltrials_20211014_shard1_replica_n35",
"shard":"shard1",
"collection":"clinicaltrials_20211014",
"node_name":"solr-0001:8983_solr",
"type":"NRT",
"leader":"true",
"base_url":"http://solr-0001:8983/solr";,
"state":"active",
"force_set_state":"false",
"INDEX.sizeInGB":8.830416035838425}}]}}}
,{
"node":"solr-0004:8983_solr",
"isLive":true,
"cores":5.0,
"freedisk":965.6229667663574,
"sysLoadAvg":16.0,
"totaldisk":1151.609375,
"replicas":{
"alldata":{"shard2":[{
"core_node68":{
"core":"alldata_shard2_replica_n67",
"shard":"shard2",
"collection":"alldata",
"node_name":"solr-0004:8983_solr",
"type":"NRT",
"leader":"true",
"base_url":"http://solr-0004:8983/solr";,
"state":"active",
"force_set_state":"false",
"INDEX.sizeInGB":174.74850487802178}}]},
".system":{"shard1":[{
"core_node3":{
"core":".system_shard1_replica_n1",
"shard":"shard1",
"collection":".system",
"node_name":"solr-0004:8983_solr",
"type":"NRT",
"leader":"true",
"base_url":"http://solr-0004:8983/solr";,
"state":"active",
"force_set_state":"false",
"INDEX.sizeInGB":6.414949893951416E-5}}]},
"weeklies_2021-10-25_7_7_3":{
"shard2":[],
"shard1":[{
"core_node168":{
"core":"weeklies_2021-10-25_7_7_3_shard1_replica_n167",
"shard":"shard1",
"collection":"weeklies_2021-10-25_7_7_3",
"node_name":"solr-0004:8983_solr",
"type":"NRT",
"base_url":"http://solr-0004:8983/solr";,
"state":"recovery_failed",
"force_set_state":"false",
"INDEX.sizeInGB":6.426125764846802E-8}}]},
"dailies_2021-1":{"shard1":[{
"core_node34":{
"core":"dailies_2021-1_shard1_replica_n33",
"shard":"shard1",
"collection":"dailies_2021-1",
"node_name":"solr-0004:8983_solr",
"type":"NRT",
"leader":"true",
"base_url":"http://solr-0004:8983/solr";,
"state":"active",
"force_set_state":"false",
"INDEX.sizeInGB":0.46995995100587606}}]},
"clinicaltrials_20211014":{"shard2":[{
"core_node24":{
"core":"clinicaltrials_20211014_shard2_replica_n23",
"shard":"shard2",
"collection":"clinicaltrials_20211014",
"node_name":"solr-0004:8983_solr",
"type":"NRT",
"leader":"true",
"base_url":"http://solr-0004:8983/solr";,
"state":"active",
"force_set_state":"false",
"INDEX.sizeInGB":9.058873846195638}}]}}}
,{
"node":"solr-0003:8983_solr",
"isLive":true,
"cores":4.0,
"freedisk":942.7971115112305,
"sysLoadAvg":3.0,
"totaldisk":1151.609375,
"replicas":{
"alldata":{"shard2":[{
"solr-0003.alldata":{
"core":"alldata",
"shard":"shard2",
"collection":"alldata",
"node_name":"solr-0003:8983_solr",
"type":"NRT",
"base_url":"http://solr-0003:8983/solr";,
"state":"active",
"force_set_state":"false"}}]},
"weeklies_2021-10-25_7_7_3":{
"shard2":[],
"shard1":[]},
".system":{"shard1":[{
"core_node5":{
"core":".system_shard1_replica_n2",
"shard":"shard1",
"collection":".system",
"node_name":"solr-0003:8983_solr",
"type":"NRT",
"base_url":"http://solr-0003:8983/solr";,
"state":"active",
"force_set_state":"false",
"INDEX.sizeInGB":6.421282887458801E-5}}]},
"dailies_2021-1":{"shard1":[{
"core_node3":{
"core":"dailies_2021-1_shard1_replica_n1",
"shard":"shard1",
"collection":"dailies_2021-1",
"node_name":"solr-0003:8983_solr",
"type":"NRT",
"base_url":"http://solr-0003:8983/solr";,
"state":"active",
"force_set_state":"false",
"INDEX.sizeInGB":0.46995995100587606}}]},
"clinicaltrials_20211014":{"shard2":[{
"core_node22":{
"core":"clinicaltrials_20211014_shard2_replica_n21",
"shard":"shard2",
"collection":"clinicaltrials_20211014",
"node_name":"solr-0003:8983_solr",
"type":"NRT",
"base_url":"http://solr-0003:8983/solr";,
"state":"active",
"force_set_state":"false",
"INDEX.sizeInGB":9.058923612348735}}]}}}],
"liveNodes":["solr-0001:8983_solr",
"solr-0002:8983_solr",
"solr-0003:8983_solr",
"solr-0004:8983_solr"],
"violations":[],
"config":{
"cluster-preferences":[{
"minimize":"cores",
"precision":2}
,{
"maximize":"freedisk",
"precision":100}
,{
"minimize":"sysLoadAvg",
"precision":10}],
"cluster-policy":[{
"strict":false,
"replica":"<2",
"shard":"#EACH",
"node":"#ANY"}]}}
2021-11-08 00:43:38.287 ERROR
(OverseerThreadFactory-38-thread-5-processing-n:solr-0001:8983_solr)
[c:weeklies_2021-10-25_7_7_3 s:shard2 ]
o.a.s.c.a.c.OverseerCollectionMessageHandler Collection:
weeklies_2021-10-25_7_7_3 operation: addreplica
failed:org.apache.solr.cloud.api.collections.Assign$AssignmentException:
Error getting replica locations : No node can satisfy the rules
"[{strict=false, replica=<2, shard=#EACH, node=#ANY,
collection=alldata}, {strict=false, replica=<2, shard=#EACH, node=#ANY,
collection=weeklies_2021-10-25_7_7_3}, {strict=false, replica=<2,
shard=#EACH, node=#ANY, collection=.system}, {strict=false, replica=<2,
shard=#EACH, node=#ANY, collection=dailies_2021-1}, {strict=false,
replica=<2, shard=#EACH, node=#ANY, collection=clinicaltrials_20211014}]
More details from logs in node : solr-0001:8983_solr, errorId :
AutoScaling.error.diagnostics.2010951363830029"
at
org.apache.solr.cloud.api.collections.Assign.getPositionsUsingPolicy(Assign.java:394)
at
org.apache.solr.cloud.api.collections.Assign$PolicyBasedAssignStrategy.assign(Assign.java:630)
at
org.apache.solr.cloud.api.collections.Assign.getNodesForNewReplicas(Assign.java:368)
at
org.apache.solr.cloud.api.collections.AddReplicaCmd.buildReplicaPositions(AddReplicaCmd.java:360)
at
org.apache.solr.cloud.api.collections.AddReplicaCmd.addReplica(AddReplicaCmd.java:146)
at
org.apache.solr.cloud.api.collections.AddReplicaCmd.call(AddReplicaCmd.java:91)
at
org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:294)
at
org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:505)
at
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:829)
Caused by: org.apache.solr.common.SolrException: No node can satisfy
the rules "[{strict=false, replica=<2, shard=#EACH, node=#ANY,
collection=alldata}, {strict=false, replica=<2, shard=#EACH, node=#ANY,
collection=weeklies_2021-10-25_7_7_3}, {strict=false, replica=<2,
shard=#EACH, node=#ANY, collection=.system}, {strict=false, replica=<2,
shard=#EACH, node=#ANY, collection=dailies_2021-1}, {strict=false,
replica=<2, shard=#EACH, node=#ANY, collection=clinicaltrials_20211014}]
More details from logs in node : solr-0001:8983_solr, errorId :
AutoScaling.error.diagnostics.2010951363830029"
at
org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper.getReplicaLocations(PolicyHelper.java:179)
at
org.apache.solr.cloud.api.collections.Assign.getPositionsUsingPolicy(Assign.java:382)
... 11 more
