[
https://issues.apache.org/jira/browse/IGNITE-18693?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Mikhail Petrov updated IGNITE-18693:
------------------------------------
Description:
Exception raised eventually after cluster activation:
Logs and config are in attach.
{noformat}
Failed to find security context for subject with given ID :
32fa7141-69fd-4c63-a3c6-9604bb2ff6cf]]
java.lang.IllegalStateException: Failed to find security context for subject
with given ID : 32fa7141-69fd-4c63-a3c6-9604bb2ff6cf
at
org.apache.ignite.internal.processors.security.IgniteSecurityProcessor.withContext(IgniteSecurityProcessor.java:167)
at
org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1906)
at
org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1528)
at
org.apache.ignite.internal.managers.communication.GridIoManager.access$5300(GridIoManager.java:243)
at
org.apache.ignite.internal.managers.communication.GridIoManager$9.execute(GridIoManager.java:1421)
at
org.apache.ignite.internal.managers.communication.TraceRunnable.run(TraceRunnable.java:55)
at
org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:637)
at
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:125)
at java.lang.Thread.run(Thread.java:748)
{noformat}
Possible reproducer:
{code:java}
public class NodeSecurityContextTest extends AbstractSecurityTest {
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String
igniteInstanceName) throws Exception {
return getConfiguration(igniteInstanceName, false);
}
/** */
private IgniteConfiguration getConfiguration(String igniteInstanceName,
boolean isClient) throws Exception {
return super.getConfiguration(igniteInstanceName)
.setFailureHandler(new StopNodeOrHaltFailureHandler())
.setDataStorageConfiguration(new DataStorageConfiguration())
.setClientMode(isClient)
.setDataStorageConfiguration(new DataStorageConfiguration()
.setDefaultDataRegionConfiguration(new DataRegionConfiguration()
.setPersistenceEnabled(true)
.setMaxSize(100L * 1024 * 1024)))
.setAuthenticationEnabled(true);
}
/** */
private IgniteEx startClientNode(int idx) throws Exception {
return startGrid(getConfiguration(getTestIgniteInstanceName(idx),
true));
}
/** */
@Test
public void testClientNodeDiscoveryNotificationWorkerHanging() throws
Exception {
startGrids(2).cluster().state(ACTIVE);
grid(0).createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME)
.setBackups(1)
.setWriteSynchronizationMode(FULL_SYNC)
.setAtomicityMode(TRANSACTIONAL));
awaitPartitionMapExchange();
CountDownLatch notificationWrkBlockedLatch = new CountDownLatch(1);
grid(1).context().io().addMessageListener(GridTopic.TOPIC_CACHE,
(nodeId, msg, plc) -> {
if (msg instanceof GridDhtTxPrepareRequest)
notificationWrkBlockedLatch.countDown();
});
blockDiscoveryNotificationWorker(grid(1), notificationWrkBlockedLatch);
IgniteEx cli = startGrid(getConfiguration(getTestIgniteInstanceName(2),
true));
IgniteCache<Integer, Integer> cache = cli.cache(DEFAULT_CACHE_NAME);
int key = keyForNode(grid(0).affinity(DEFAULT_CACHE_NAME), new
AtomicInteger(), grid(0).localNode());
cache.put(key, 0);
}
/** */
private void blockDiscoveryNotificationWorker(IgniteEx ignite,
CountDownLatch latch) throws Exception {
Object discoWrk = U.field(ignite.context().discovery(), "discoNtfWrk");
Method submitMethod = discoWrk.getClass().getDeclaredMethod("submit",
GridFutureAdapter.class, Runnable.class);
submitMethod.setAccessible(true);
Runnable blockTask = () -> {
try {
assertTrue(latch.await(getTestTimeout(),
TimeUnit.MILLISECONDS));
}
catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new IgniteException(e);
}
};
submitMethod.invoke(discoWrk, new GridFutureAdapter<>(), blockTask);
}
}
{code}
Possible cause of the failure:
1. A client/server node joined the cluster, but a particular server node has
not yet updated its topology snapshot.
2. The mentioned server node receives a communication message from the joining
node, so the receiver node needs to switch its security context to the sender
node one (see GridIoManager#invokeListener and
IgniteSecurityProcessor#withContext(java.util.UUID))
3. During IgniteSecurityProcessor#withContext execution of
{code:java}
ClusterNode node = Optional.ofNullable(ctx.discovery().node(subjId))
.orElseGet(() -> ctx.discovery().historicalNode(subjId));
{code}
returns null because the current topology snapshot does not include the
joining node yet.
4. When we failed to find message sender node id in the topology, we explicitly
ask the security plugin to provide security context, but it returns null
because current implementation of AuthenticationProcessor is aware only of thin
client secrity contexts.
5. So we fail on this check
{code:java}
if (res == null) {
throw new IllegalStateException("Failed to find security
context " +
"for subject with given ID : " + subjId);
{code}
was:
Exception raised eventually after cluster activation:
Logs and config are in attach.
{noformat}
Failed to find security context for subject with given ID :
32fa7141-69fd-4c63-a3c6-9604bb2ff6cf]]
java.lang.IllegalStateException: Failed to find security context for subject
with given ID : 32fa7141-69fd-4c63-a3c6-9604bb2ff6cf
at
org.apache.ignite.internal.processors.security.IgniteSecurityProcessor.withContext(IgniteSecurityProcessor.java:167)
at
org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1906)
at
org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1528)
at
org.apache.ignite.internal.managers.communication.GridIoManager.access$5300(GridIoManager.java:243)
at
org.apache.ignite.internal.managers.communication.GridIoManager$9.execute(GridIoManager.java:1421)
at
org.apache.ignite.internal.managers.communication.TraceRunnable.run(TraceRunnable.java:55)
at
org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:637)
at
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:125)
at java.lang.Thread.run(Thread.java:748)
{noformat}
Possible reproducer:
{code:java}
public class NodeSecurityContextTest extends AbstractSecurityTest {
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String
igniteInstanceName) throws Exception {
return getConfiguration(igniteInstanceName, false);
}
/** */
private IgniteConfiguration getConfiguration(String igniteInstanceName,
boolean isClient) throws Exception {
return super.getConfiguration(igniteInstanceName)
.setFailureHandler(new StopNodeOrHaltFailureHandler())
.setDataStorageConfiguration(new DataStorageConfiguration())
.setClientMode(isClient)
.setDataStorageConfiguration(new DataStorageConfiguration()
.setDefaultDataRegionConfiguration(new DataRegionConfiguration()
.setPersistenceEnabled(true)
.setMaxSize(100L * 1024 * 1024)))
.setAuthenticationEnabled(true);
}
/** */
private IgniteEx startClientNode(int idx) throws Exception {
return startGrid(getConfiguration(getTestIgniteInstanceName(idx),
true));
}
/** */
@Test
public void testClientNodeDiscoveryNotificationWorkerHanging() throws
Exception {
startGrids(2).cluster().state(ACTIVE);
grid(0).createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME)
.setBackups(1)
.setWriteSynchronizationMode(FULL_SYNC)
.setAtomicityMode(TRANSACTIONAL));
awaitPartitionMapExchange();
CountDownLatch notificationWrkBlockedLatch = new CountDownLatch(1);
grid(1).context().io().addMessageListener(GridTopic.TOPIC_CACHE,
(nodeId, msg, plc) -> {
if (msg instanceof GridDhtTxPrepareRequest)
notificationWrkBlockedLatch.countDown();
});
blockDiscoveryNotificationWorker(grid(1), notificationWrkBlockedLatch);
IgniteEx cli = startGrid(getConfiguration(getTestIgniteInstanceName(2),
true));
IgniteCache<Integer, Integer> cache = cli.cache(DEFAULT_CACHE_NAME);
int key = keyForNode(grid(0).affinity(DEFAULT_CACHE_NAME), new
AtomicInteger(), grid(0).localNode());
cache.put(key, 0);
}
/** */
private void blockDiscoveryNotificationWorker(IgniteEx ignite,
CountDownLatch latch) throws Exception {
Object discoWrk = U.field(ignite.context().discovery(), "discoNtfWrk");
Method submitMethod = discoWrk.getClass().getDeclaredMethod("submit",
GridFutureAdapter.class, Runnable.class);
submitMethod.setAccessible(true);
Runnable blockTask = () -> {
try {
assertTrue(latch.await(getTestTimeout(),
TimeUnit.MILLISECONDS));
}
catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new IgniteException(e);
}
};
submitMethod.invoke(discoWrk, new GridFutureAdapter<>(), blockTask);
}
}
{code}
Possible cause of the failure:
1. A client/server node joined the cluster, but a particular server node has
not yet updated its topology snapshot.
2. The mentioned server node receives a communication message from the joining
node, so the receiver node needs to switch its security context to the sender
node one (see GridIoManager#invokeListener and
IgniteSecurityProcessor#withContext(java.util.UUID))
3. During IgniteSecurityProcessor#withContext execution of
{code:java}
ClusterNode node = Optional.ofNullable(ctx.discovery().node(subjId))
.orElseGet(() -> ctx.discovery().historicalNode(subjId));
{code}
returns null as the current topology snapshot does not include the joining
node yet.
4. When we failed to find message sender node id in the topology, we explicitly
ask the security plugin to provide security context, but it returns null as
current implementation of AuthenticationProcessor is aware only of thin client
secrity contexts.
5. So we fail on this check
{code:java}
if (res == null) {
throw new IllegalStateException("Failed to find security
context " +
"for subject with given ID : " + subjId);
{code}
> Failed to find security context for subject with given ID.
> ----------------------------------------------------------
>
> Key: IGNITE-18693
> URL: https://issues.apache.org/jira/browse/IGNITE-18693
> Project: Ignite
> Issue Type: Bug
> Components: security
> Affects Versions: 2.14
> Reporter: Evgeny Stanilovsky
> Priority: Major
> Attachments: ignite (2).log, test-config.xml
>
>
> Exception raised eventually after cluster activation:
> Logs and config are in attach.
> {noformat}
> Failed to find security context for subject with given ID :
> 32fa7141-69fd-4c63-a3c6-9604bb2ff6cf]]
> java.lang.IllegalStateException: Failed to find security context for subject
> with given ID : 32fa7141-69fd-4c63-a3c6-9604bb2ff6cf
> at
> org.apache.ignite.internal.processors.security.IgniteSecurityProcessor.withContext(IgniteSecurityProcessor.java:167)
> at
> org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1906)
> at
> org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1528)
> at
> org.apache.ignite.internal.managers.communication.GridIoManager.access$5300(GridIoManager.java:243)
> at
> org.apache.ignite.internal.managers.communication.GridIoManager$9.execute(GridIoManager.java:1421)
> at
> org.apache.ignite.internal.managers.communication.TraceRunnable.run(TraceRunnable.java:55)
> at
> org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:637)
> at
> org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:125)
> at java.lang.Thread.run(Thread.java:748)
> {noformat}
> Possible reproducer:
> {code:java}
> public class NodeSecurityContextTest extends AbstractSecurityTest {
> /** {@inheritDoc} */
> @Override protected IgniteConfiguration getConfiguration(String
> igniteInstanceName) throws Exception {
> return getConfiguration(igniteInstanceName, false);
> }
> /** */
> private IgniteConfiguration getConfiguration(String igniteInstanceName,
> boolean isClient) throws Exception {
> return super.getConfiguration(igniteInstanceName)
> .setFailureHandler(new StopNodeOrHaltFailureHandler())
> .setDataStorageConfiguration(new DataStorageConfiguration())
> .setClientMode(isClient)
> .setDataStorageConfiguration(new DataStorageConfiguration()
> .setDefaultDataRegionConfiguration(new
> DataRegionConfiguration()
> .setPersistenceEnabled(true)
> .setMaxSize(100L * 1024 * 1024)))
> .setAuthenticationEnabled(true);
> }
> /** */
> private IgniteEx startClientNode(int idx) throws Exception {
> return startGrid(getConfiguration(getTestIgniteInstanceName(idx),
> true));
> }
> /** */
> @Test
> public void testClientNodeDiscoveryNotificationWorkerHanging() throws
> Exception {
> startGrids(2).cluster().state(ACTIVE);
> grid(0).createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME)
> .setBackups(1)
> .setWriteSynchronizationMode(FULL_SYNC)
> .setAtomicityMode(TRANSACTIONAL));
> awaitPartitionMapExchange();
> CountDownLatch notificationWrkBlockedLatch = new CountDownLatch(1);
> grid(1).context().io().addMessageListener(GridTopic.TOPIC_CACHE,
> (nodeId, msg, plc) -> {
> if (msg instanceof GridDhtTxPrepareRequest)
> notificationWrkBlockedLatch.countDown();
> });
> blockDiscoveryNotificationWorker(grid(1),
> notificationWrkBlockedLatch);
> IgniteEx cli =
> startGrid(getConfiguration(getTestIgniteInstanceName(2), true));
> IgniteCache<Integer, Integer> cache = cli.cache(DEFAULT_CACHE_NAME);
> int key = keyForNode(grid(0).affinity(DEFAULT_CACHE_NAME), new
> AtomicInteger(), grid(0).localNode());
> cache.put(key, 0);
> }
> /** */
> private void blockDiscoveryNotificationWorker(IgniteEx ignite,
> CountDownLatch latch) throws Exception {
> Object discoWrk = U.field(ignite.context().discovery(),
> "discoNtfWrk");
> Method submitMethod = discoWrk.getClass().getDeclaredMethod("submit",
> GridFutureAdapter.class, Runnable.class);
> submitMethod.setAccessible(true);
> Runnable blockTask = () -> {
> try {
> assertTrue(latch.await(getTestTimeout(),
> TimeUnit.MILLISECONDS));
> }
> catch (InterruptedException e) {
> Thread.currentThread().interrupt();
> throw new IgniteException(e);
> }
> };
> submitMethod.invoke(discoWrk, new GridFutureAdapter<>(), blockTask);
> }
> }
> {code}
> Possible cause of the failure:
> 1. A client/server node joined the cluster, but a particular server node has
> not yet updated its topology snapshot.
> 2. The mentioned server node receives a communication message from the
> joining node, so the receiver node needs to switch its security context to
> the sender node one (see GridIoManager#invokeListener and
> IgniteSecurityProcessor#withContext(java.util.UUID))
> 3. During IgniteSecurityProcessor#withContext execution of
> {code:java}
> ClusterNode node = Optional.ofNullable(ctx.discovery().node(subjId))
> .orElseGet(() -> ctx.discovery().historicalNode(subjId));
> {code}
> returns null because the current topology snapshot does not include the
> joining node yet.
> 4. When we failed to find message sender node id in the topology, we
> explicitly ask the security plugin to provide security context, but it
> returns null because current implementation of AuthenticationProcessor is
> aware only of thin client secrity contexts.
> 5. So we fail on this check
> {code:java}
> if (res == null) {
> throw new IllegalStateException("Failed to find security
> context " +
> "for subject with given ID : " + subjId);
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
