Hi Chris,
> Date: Mon, 10 Jun 2013 10:18:10 -0400
> From: ch...@christopherschultz.net
> To: users@tomcat.apache.org
> Subject: Re: Illegal access: this web application instance has been stopped
> already and NoClassDefFoundError
>
> -----BEGIN PGP SIGNED MESSAGE-----
> Hash: SHA256
>
> Bob,
>
> On 6/8/13 11:14 PM, ruxing bao wrote:
> > I'am using Zookeeper client API in java web application.When I shut
> > down tomcat,I am getting this error :
> >
> >
> >
> > 2013-06-07 12:01:29,676 ERROR [org.apache.zookeeper.ClientCnxn]
> > (xxx-startStop-1-SendThread(xxx.net:12183)) from
> > xxx-startStop-1-SendThread(xxx.net:12183)
> > java.lang.NoClassDefFoundError:
> > org/apache/zookeeper/server/ZooTrace at
> > org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1115)
> >
> >
> Caused by: java.lang.ClassNotFoundException:
> org.apache.zookeeper.server.ZooTrace
> > at
> > org.apache.catalina.loader.WebappClassLoader.loadClass(WebappClassLoader.java:1711)
> >
> >
> at
> org.apache.catalina.loader.WebappClassLoader.loadClass(WebappClassLoader.java:1556)
> > ... 1 more
> >
> >
> >
> >
> > at the same time,the following is also in catalina.out:
> >
> >
> >
> > INFO: Illegal access: this web application instance has been
> > stopped already. Could not load
> > org.apache.zookeeper.server.ZooTrace. The eventual following stack
> > trace is caused by an error thrown for debugging purposes as well
> > as to attempt to terminate the thread which caused the illegal
> > access, and has no functional impact.
> > java.lang.IllegalStateException at
> > org.apache.catalina.loader.WebappClassLoader.loadClass(WebappClassLoader.java:1597)
> >
> >
> at
> org.apache.catalina.loader.WebappClassLoader.loadClass(WebappClassLoader.java:1556)
> > at
> > org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1115)
> >
> >
> >
> >
> > I searched google and mail list ,couldn't find any solution,please
> > help me.
> >
> >
> > Enviroment info:
> >
> > JDK:1.6.0_45
> >
> > Tomcat:7.0.40
> >
> > Zookeeper:3.4.5
>
> Can you give us any more of the stack trace? It looks like this is
> happening during shutdown, but all the ServletContextListeners should
> complete before the WebappClassLoader starts shedding its loaded classes.
>
> Are you explicitly shutting-down the ClientCnxn thread in a SCL's
> destroy() method? If not, you need to do that.
>
Sorry,I can't get any more of the stack trace.
We wrapped zookeepr client as a spring bean and invoked method "close" of
zookeeper in "destory-method" of bean,in that method "close",zookeeper Send
Thread was closed.
When tomcat was shut down, Spring closed it's container and bean was detroyed,
then "destroy-method" of bean was invoked.
I'am confusing why is Send Thread of Zookeeper seemed to exit slower than the
WebappClassLoader shedding its loaded classes.
ps: details of zookeeper closing
org.apache.zookeeper.Zookeeper
public synchronized void close() throws InterruptedException {
if (!cnxn.getState().isAlive()) {
if (LOG.isDebugEnabled()) {
LOG.debug("Close called on already closed client");
}
return;
}
if (LOG.isDebugEnabled()) {
LOG.debug("Closing session: 0x" + Long.toHexString(getSessionId()));
}
try {
cnxn.close();
} catch (IOException e) {
if (LOG.isDebugEnabled()) {
LOG.debug("Ignoring unexpected exception during close", e);
}
}
LOG.info("Session: 0x" + Long.toHexString(getSessionId()) + " closed");
}
----------------------------------------------------------------------------------------------------------------------
org.apache.zookeeper.ClientCnxn
public void close() throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("Closing client for session: 0x"
+ Long.toHexString(getSessionId()));
}
try {
RequestHeader h = new RequestHeader();
h.setType(ZooDefs.OpCode.closeSession);
submitRequest(h, null, null, null);
} catch (InterruptedException e) {
// ignore, close the send/event threads
} finally {
disconnect();
}
}
public void disconnect() {
if (LOG.isDebugEnabled()) {
LOG.debug("Disconnecting client for session: 0x"
+ Long.toHexString(getSessionId()));
}
sendThread.close();
eventThread.queueEventOfDeath();
}
-------------------------------------------------------------------------------------------------
org.apache.zookeeper.ClientCnxn.SendThread
void close() {
state = States.CLOSED;
clientCnxnSocket.wakeupCnxn();
}
@Override
public void run() {
clientCnxnSocket.introduce(this,sessionId);
clientCnxnSocket.updateNow();
clientCnxnSocket.updateLastSendAndHeard();
int to;
long lastPingRwServer = System.currentTimeMillis();
while (state.isAlive()) {
try {
if (!clientCnxnSocket.isConnected()) {
if(!isFirstConnect){
try {
Thread.sleep(r.nextInt(1000));
} catch (InterruptedException e) {
LOG.warn("Unexpected exception", e);
}
}
// don't re-establish connection if we are closing
if (closing || !state.isAlive()) {
break;
}
startConnect();
clientCnxnSocket.updateLastSendAndHeard();
}
if (state.isConnected()) {
// determine whether we need to send an AuthFailed
event.
if (zooKeeperSaslClient != null) {
boolean sendAuthEvent = false;
if (zooKeeperSaslClient.getSaslState() ==
ZooKeeperSaslClient.SaslState.INITIAL) {
try {
zooKeeperSaslClient.initialize(ClientCnxn.this);
} catch (SaslException e) {
LOG.error("SASL authentication with
Zookeeper Quorum member failed: " + e);
state = States.AUTH_FAILED;
sendAuthEvent = true;
}
}
KeeperState authState =
zooKeeperSaslClient.getKeeperState();
if (authState != null) {
if (authState == KeeperState.AuthFailed) {
// An authentication error occurred during
authentication with the Zookeeper Server.
state = States.AUTH_FAILED;
sendAuthEvent = true;
} else {
if (authState ==
KeeperState.SaslAuthenticated) {
sendAuthEvent = true;
}
}
}
if (sendAuthEvent == true) {
eventThread.queueEvent(new WatchedEvent(
Watcher.Event.EventType.None,
authState,null));
}
}
to = readTimeout - clientCnxnSocket.getIdleRecv();
} else {
to = connectTimeout - clientCnxnSocket.getIdleRecv();
}
if (to <= 0) {
throw new SessionTimeoutException(
"Client session timed out, have not heard from
server in "
+ clientCnxnSocket.getIdleRecv() + "ms"
+ " for sessionid 0x"
+ Long.toHexString(sessionId));
}
if (state.isConnected()) {
int timeToNextPing = readTimeout / 2
- clientCnxnSocket.getIdleSend();
if (timeToNextPing <= 0) {
sendPing();
clientCnxnSocket.updateLastSend();
} else {
if (timeToNextPing < to) {
to = timeToNextPing;
}
}
}
// If we are in read-only mode, seek for read/write server
if (state == States.CONNECTEDREADONLY) {
long now = System.currentTimeMillis();
int idlePingRwServer = (int) (now - lastPingRwServer);
if (idlePingRwServer >= pingRwTimeout) {
lastPingRwServer = now;
idlePingRwServer = 0;
pingRwTimeout =
Math.min(2*pingRwTimeout, maxPingRwTimeout);
pingRwServer();
}
to = Math.min(to, pingRwTimeout - idlePingRwServer);
}
clientCnxnSocket.doTransport(to, pendingQueue,
outgoingQueue, ClientCnxn.this);
} catch (Throwable e) {
if (closing) {
if (LOG.isDebugEnabled()) {
// closing so this is expected
LOG.debug("An exception was thrown while closing
send thread for session 0x"
+ Long.toHexString(getSessionId())
+ " : " + e.getMessage());
}
break;
} else {
// this is ugly, you have a better way speak up
if (e instanceof SessionExpiredException) {
LOG.info(e.getMessage() + ", closing socket
connection");
} else if (e instanceof SessionTimeoutException) {
LOG.info(e.getMessage() + RETRY_CONN_MSG);
} else if (e instanceof EndOfStreamException) {
LOG.info(e.getMessage() + RETRY_CONN_MSG);
} else if (e instanceof RWServerFoundException) {
LOG.info(e.getMessage());
} else {
LOG.warn(
"Session 0x"
+ Long.toHexString(getSessionId())
+ " for server "
+
clientCnxnSocket.getRemoteSocketAddress()
+ ", unexpected error"
+ RETRY_CONN_MSG, e);
}
cleanup();
if (state.isAlive()) {
eventThread.queueEvent(new WatchedEvent(
Event.EventType.None,
Event.KeeperState.Disconnected,
null));
}
clientCnxnSocket.updateNow();
clientCnxnSocket.updateLastSendAndHeard();
}
}
}
cleanup();
clientCnxnSocket.close();
if (state.isAlive()) {
eventThread.queueEvent(new WatchedEvent(Event.EventType.None,
Event.KeeperState.Disconnected, null));
}
ZooTrace.logTraceMessage(LOG, ZooTrace.getTextTraceLevel(),
"SendThread exitedloop.");//at this line,
error occurs.
}
--------------------------------------------------------------------------------------------------------------
Regards,
Bob
