While our users were adding lots of content through the document library/image gallery we noticed that at a point the appserver database resource pool became exhausted and the appserver started denying service. After a bit of digging, jstack told us that two threads were in deadlock (listing below) and most of the other threads were waiting to lock org.apache.jackrabbit.core.version.VersionManagerImpl$DynamicESCFactory which was being held by the following thread:
"httpSSLWorkerThread-48080-7" daemon prio=3 tid=0x0000000101820800 nid=0x2eb in Object.wait() [0xfffffffe413f7000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0xffffffff195cc290> (a EDU.oswego.cs.dl.util.concurrent.WriterPreferenceReadWriteLock$WriterLock)
at java.lang.Object.wait(Object.java:485)
at EDU.oswego.cs.dl.util.concurrent.WriterPreferenceReadWriteLock$WriterLock.acquire(Unknown Source)- locked <0xffffffff195cc290> (a EDU.oswego.cs.dl.util.concurrent.WriterPreferenceReadWriteLock$WriterLock)
at org.apache.jackrabbit.core.journal.AbstractJournal.lockAndSync(AbstractJournal.java:245)
at org.apache.jackrabbit.core.journal.DefaultRecordProducer.append(DefaultRecordProducer.java:51)
at org.apache.jackrabbit.core.cluster.ClusterNode$WorkspaceUpdateChannel.updateCreated(ClusterNode.java:602)
at org.apache.jackrabbit.core.state.SharedItemStateManager$Update.begin(SharedItemStateManager.java:554)
at org.apache.jackrabbit.core.state.SharedItemStateManager.beginUpdate(SharedItemStateManager.java:1085)
at org.apache.jackrabbit.core.state.SharedItemStateManager.update(SharedItemStateManager.java:1115)
at org.apache.jackrabbit.core.state.LocalItemStateManager.update(LocalItemStateManager.java:351)
at org.apache.jackrabbit.core.state.LocalItemStateManager.update(LocalItemStateManager.java:326)
at org.apache.jackrabbit.core.version.AbstractVersionManager$WriteOperation.save(AbstractVersionManager.java:189)
at org.apache.jackrabbit.core.version.AbstractVersionManager.createVersionHistory(AbstractVersionManager.java:369)
at org.apache.jackrabbit.core.version.VersionManagerImpl$1.run(VersionManagerImpl.java:221)
at org.apache.jackrabbit.core.version.VersionManagerImpl$DynamicESCFactory.doSourced(VersionManagerImpl.java:586)- locked <0xffffffff4f386e70> (a org.apache.jackrabbit.core.version.VersionManagerImpl$DynamicESCFactory)
at org.apache.jackrabbit.core.version.VersionManagerImpl.createVersionHistory(VersionManagerImpl.java:218)
at org.apache.jackrabbit.core.version.XAVersionManager.createVersionHistory(XAVersionManager.java:157)
at org.apache.jackrabbit.core.version.AbstractVersionManager.getVersionHistory(AbstractVersionManager.java:273)
at org.apache.jackrabbit.core.ItemImpl.initVersionHistories(ItemImpl.java:738)
at org.apache.jackrabbit.core.ItemImpl.save(ItemImpl.java:1088)- locked <0xffffffff501961f8> (a org.apache.jackrabbit.core.XASessionImpl)
at org.apache.jackrabbit.core.SessionImpl.save(SessionImpl.java:915)
at com.liferay.documentlibrary.util.JCRHook.addFile(JCRHook.java:143)
at com.liferay.documentlibrary.util.HookProxyImpl.addFile(HookProxyImpl.java:74)
at com.liferay.documentlibrary.util.SafeFileNameHookWrapper.addFile(SafeFileNameHookWrapper.java:100)
at com.liferay.documentlibrary.service.impl.DLLocalServiceImpl.addFile(DLLocalServiceImpl.java:71)
at sun.reflect.GeneratedMethodAccessor1419.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:309)
at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:110)
at com.liferay.portal.dao.jdbc.aop.DynamicDataSourceTransactionInterceptor.invoke(DynamicDataSourceTransactionInterceptor.java:44)
at com.liferay.portal.spring.aop.ChainableMethodAdvice.invoke(ChainableMethodAdvice.java:58)
at com.liferay.portal.spring.aop.ChainableMethodAdvice.invoke(ChainableMethodAdvice.java:58)
at com.liferay.portal.spring.aop.ChainableMethodAdvice.invoke(ChainableMethodAdvice.java:58)
Example of thread waiting on the above lock:
"httpSSLWorkerThread-48080-0" daemon prio=3 tid=0x0000000103623000 nid=0x2e4 waiting for monitor entry [0xfffffffe441f4000]
java.lang.Thread.State: BLOCKED (on object monitor)
at org.apache.jackrabbit.core.version.VersionManagerImpl$DynamicESCFactory.doSourced(VersionManagerImpl.java:584)
- waiting to lock <0xffffffff4f386e70> (a org.apache.jackrabbit.core.version.VersionManagerImpl$DynamicESCFactory)
at org.apache.jackrabbit.core.version.VersionManagerImpl.createVersionHistory(VersionManagerImpl.java:218)
at org.apache.jackrabbit.core.version.XAVersionManager.createVersionHistory(XAVersionManager.java:157)
at org.apache.jackrabbit.core.version.AbstractVersionManager.getVersionHistory(AbstractVersionManager.java:273)
at org.apache.jackrabbit.core.ItemImpl.initVersionHistories(ItemImpl.java:738)
at org.apache.jackrabbit.core.ItemImpl.save(ItemImpl.java:1088)- locked <0xffffffff4f20a6d8> (a org.apache.jackrabbit.core.XASessionImpl)
at org.apache.jackrabbit.core.SessionImpl.save(SessionImpl.java:915)
at com.liferay.documentlibrary.util.JCRHook.addFile(JCRHook.java:143)
at com.liferay.documentlibrary.util.HookProxyImpl.addFile(HookProxyImpl.java:74)
at com.liferay.documentlibrary.util.SafeFileNameHookWrapper.addFile(SafeFileNameHookWrapper.java:100)
at com.liferay.documentlibrary.service.impl.DLLocalServiceImpl.addFile(DLLocalServiceImpl.java:71)
at sun.reflect.GeneratedMethodAccessor1419.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
Deadlock thread dump:
Java stack information for the threads listed above:
===================================================
"httpSSLWorkerThread-48080-21":
at org.apache.jackrabbit.core.state.NodeState.getChildNodeEntry(NodeState.java:300)
- waiting to lock <0xffffffff502af0f0> (a org.apache.jackrabbit.core.state.NodeState)
at org.apache.jackrabbit.core.CachingHierarchyManager.nodeModified(CachingHierarchyManager.java:316)- locked <0xffffffff4fbc75e8> (a java.lang.Object)
at org.apache.jackrabbit.core.CachingHierarchyManager.stateModified(CachingHierarchyManager.java:293)
at org.apache.jackrabbit.core.state.StateChangeDispatcher.notifyStateModified(StateChangeDispatcher.java:111)
at org.apache.jackrabbit.core.state.SessionItemStateManager.stateModified(SessionItemStateManager.java:889)
at org.apache.jackrabbit.core.state.StateChangeDispatcher.notifyStateModified(StateChangeDispatcher.java:111)
at org.apache.jackrabbit.core.state.LocalItemStateManager.stateModified(LocalItemStateManager.java:452)
at org.apache.jackrabbit.core.state.XAItemStateManager.stateModified(XAItemStateManager.java:602)
at org.apache.jackrabbit.core.state.StateChangeDispatcher.notifyStateModified(StateChangeDispatcher.java:111)
at org.apache.jackrabbit.core.state.SharedItemStateManager.stateModified(SharedItemStateManager.java:400)
at org.apache.jackrabbit.core.state.ItemState.notifyStateUpdated(ItemState.java:244)
at org.apache.jackrabbit.core.state.ChangeLog.persisted(ChangeLog.java:297)
at org.apache.jackrabbit.core.state.SharedItemStateManager$Update.end(SharedItemStateManager.java:749)
at org.apache.jackrabbit.core.state.SharedItemStateManager.update(SharedItemStateManager.java:1115)
at org.apache.jackrabbit.core.state.LocalItemStateManager.update(LocalItemStateManager.java:351)
at org.apache.jackrabbit.core.state.XAItemStateManager.update(XAItemStateManager.java:354)
at org.apache.jackrabbit.core.state.LocalItemStateManager.update(LocalItemStateManager.java:326)
at org.apache.jackrabbit.core.state.XAItemStateManager.update(XAItemStateManager.java:354)
at org.apache.jackrabbit.core.state.LocalItemStateManager.update(LocalItemStateManager.java:326)
at org.apache.jackrabbit.core.state.SessionItemStateManager.update(SessionItemStateManager.java:325)
at org.apache.jackrabbit.core.ItemImpl.save(ItemImpl.java:1111)- locked <0xffffffff501962d8> (a org.apache.jackrabbit.core.XASessionImpl)
at org.apache.jackrabbit.core.SessionImpl.save(SessionImpl.java:915)
at com.liferay.documentlibrary.util.JCRHook.deleteFile(JCRHook.java:351)
at com.liferay.documentlibrary.util.HookProxyImpl.deleteFile(HookProxyImpl.java:101)
at com.liferay.documentlibrary.util.SafeFileNameHookWrapper.deleteFile(SafeFileNameHookWrapper.java:146)
at com.liferay.documentlibrary.service.impl.DLServiceImpl.deleteFile(DLServiceImpl.java:105)
at sun.reflect.GeneratedMethodAccessor2144.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:309)
at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:110)
at com.liferay.portal.dao.jdbc.aop.DynamicDataSourceTransactionInterceptor.invoke(DynamicDataSourceTransactionInterceptor.java:44)
at com.liferay.portal.spring.aop.ChainableMethodAdvice.invoke(ChainableMethodAdvice.java:58)
at com.liferay.portal.spring.aop.ChainableMethodAdvice.invoke(ChainableMethodAdvice.java:58)
at com.liferay.portal.spring.aop.ChainableMethodAdvice.invoke(ChainableMethodAdvice.java:58)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:89)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:202)
at $Proxy163.deleteFile(Unknown Source)
at com.liferay.documentlibrary.service.DLServiceUtil.deleteFile(DLServiceUtil.java:73)
at com.liferay.portal.image.DLHook.updateImage(DLHook.java:93)
at com.liferay.portal.service.impl.ImageLocalServiceImpl.updateImage(ImageLocalServiceImpl.java:315)
at com.liferay.portal.service.impl.ImageLocalServiceImpl.updateImage(ImageLocalServiceImpl.java:271)
at sun.reflect.GeneratedMethodAccessor2141.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:309)
at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:110)
at com.liferay.portal.dao.jdbc.aop.DynamicDataSourceTransactionInterceptor.invoke(DynamicDataSourceTransactionInterceptor.java:44)
at com.liferay.portal.spring.aop.ChainableMethodAdvice.invoke(ChainableMethodAdvice.java:58)
at com.liferay.portal.spring.aop.ChainableMethodAdvice.invoke(ChainableMethodAdvice.java:58)
at com.liferay.portal.spring.aop.ChainableMethodAdvice.invoke(ChainableMethodAdvice.java:58)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:89)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:89)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:202)
at $Proxy38.updateImage(Unknown Source)
at com.liferay.portlet.imagegallery.service.impl.IGImageLocalServiceImpl.saveScaledImage(IGImageLocalServiceImpl.java:689)
at com.liferay.portlet.imagegallery.service.impl.IGImageLocalServiceImpl.updateSmallImage(IGImageLocalServiceImpl.java:615)
at sun.reflect.GeneratedMethodAccessor2143.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:309)
at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:110)
at com.liferay.portal.dao.jdbc.aop.DynamicDataSourceTransactionInterceptor.invoke(DynamicDataSourceTransactionInterceptor.java:44)
at com.liferay.portal.spring.aop.ChainableMethodAdvice.invoke(ChainableMethodAdvice.java:58)
at com.liferay.portal.spring.aop.ChainableMethodAdvice.invoke(ChainableMethodAdvice.java:58)
at com.liferay.portal.spring.aop.ChainableMethodAdvice.invoke(ChainableMethodAdvice.java:58)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:89)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:202)
at $Proxy214.updateSmallImage(Unknown Source)
at com.liferay.portlet.imagegallery.service.IGImageLocalServiceUtil.updateSmallImage(IGImageLocalServiceUtil.java:495)
at com.liferay.portal.servlet.ImageServlet.checkIGImageThumbnailMaxDimensions(ImageServlet.java:145)
at com.liferay.portal.servlet.ImageServlet.getImage(ImageServlet.java:223)
at com.liferay.portal.servlet.ImageServlet.writeImage(ImageServlet.java:353)
at com.liferay.portal.servlet.ImageServlet.service(ImageServlet.java:121)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:847)
at org.apache.catalina.core.ApplicationFilterChain.servletService(ApplicationFilterChain.java:427)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:333)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:214)
at com.liferay.portal.kernel.servlet.BaseFilter.processFilter(BaseFilter.java:196)
at com.liferay.portal.kernel.servlet.BaseFilter.doFilter(BaseFilter.java:126)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:246)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:214)
at com.liferay.portal.kernel.servlet.BaseFilter.processFilter(BaseFilter.java:196)
at com.liferay.portal.kernel.servlet.BaseFilter.doFilter(BaseFilter.java:126)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:246)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:214)
at com.liferay.portal.kernel.servlet.BaseFilter.processFilter(BaseFilter.java:196)
at com.liferay.portal.servlet.filters.header.HeaderFilter.processFilter(HeaderFilter.java:129)
at com.liferay.portal.kernel.servlet.BaseFilter.doFilter(BaseFilter.java:123)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:246)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:214)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:246)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:214)
at com.liferay.portal.kernel.servlet.BaseFilter.processFilter(BaseFilter.java:196)
at com.liferay.portal.kernel.servlet.BaseFilter.doFilter(BaseFilter.java:126)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:246)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:214)
at com.liferay.portal.kernel.servlet.BaseFilter.processFilter(BaseFilter.java:196)
at com.liferay.portal.kernel.servlet.BaseFilter.doFilter(BaseFilter.java:126)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:246)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:214)
at com.liferay.portal.kernel.servlet.BaseFilter.processFilter(BaseFilter.java:196)
at com.liferay.portal.servlet.filters.virtualhost.VirtualHostFilter.processFilter(VirtualHostFilter.java:239)
at com.liferay.portal.kernel.servlet.BaseFilter.doFilter(BaseFilter.java:123)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:246)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:214)
at com.liferay.portal.kernel.servlet.BaseFilter.processFilter(BaseFilter.java:196)
at com.liferay.portal.kernel.servlet.BaseFilter.doFilter(BaseFilter.java:126)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:246)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:214)
at com.liferay.portal.kernel.servlet.BaseFilter.processFilter(BaseFilter.java:196)
at com.liferay.portal.kernel.servlet.BaseFilter.doFilter(BaseFilter.java:126)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:246)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:214)
at org.tuckey.web.filters.urlrewrite.UrlRewriteFilter.doFilter(UrlRewriteFilter.java:738)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:246)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:214)
at com.liferay.portal.kernel.servlet.BaseFilter.processFilter(BaseFilter.java:196)
at com.liferay.portal.servlet.filters.threadlocal.ThreadLocalFilter.processFilter(ThreadLocalFilter.java:35)
at com.liferay.portal.kernel.servlet.BaseFilter.doFilter(BaseFilter.java:123)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:246)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:214)
at pt.segsocial.npss2.filtros.Npss2EncodingFilter.doFilter(Npss2EncodingFilter.java:47)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:246)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:214)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:313)
at org.apache.catalina.core.StandardContextValve.invokeInternal(StandardContextValve.java:287)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:218)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:648)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:593)
at com.sun.enterprise.web.WebPipeline.invoke(WebPipeline.java:94)
at com.sun.enterprise.web.PESessionLockingStandardPipeline.invoke(PESessionLockingStandardPipeline.java:98)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:222)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:648)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:593)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:648)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:593)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:587)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:1093)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:166)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:648)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:593)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:587)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:1093)
at org.apache.coyote.tomcat5.CoyoteAdapter.service(CoyoteAdapter.java:291)
at com.sun.enterprise.web.connector.grizzly.DefaultProcessorTask.invokeAdapter(DefaultProcessorTask.java:676)
at com.sun.enterprise.web.connector.grizzly.comet.CometEngine.executeServlet(CometEngine.java:664)
at com.sun.enterprise.web.connector.grizzly.comet.CometEngine.handle(CometEngine.java:409)
at com.sun.enterprise.web.connector.grizzly.comet.CometAsyncFilter.doFilter(CometAsyncFilter.java:86)
at com.sun.enterprise.web.connector.grizzly.async.DefaultAsyncExecutor.invokeFilters(DefaultAsyncExecutor.java:196)
at com.sun.enterprise.web.connector.grizzly.async.DefaultAsyncExecutor.interrupt(DefaultAsyncExecutor.java:171)
at com.sun.enterprise.web.connector.grizzly.async.AsyncProcessorTask.doTask(AsyncProcessorTask.java:96)
at com.sun.enterprise.web.connector.grizzly.TaskBase.run(TaskBase.java:269)
at com.sun.enterprise.web.connector.grizzly.ssl.SSLWorkerThread.run(SSLWorkerThread.java:111)
"httpSSLWorkerThread-48080-12":
at org.apache.jackrabbit.core.CachingHierarchyManager.nodeRemoved(CachingHierarchyManager.java:452)- waiting to lock <0xffffffff4fbc75e8> (a java.lang.Object)
at org.apache.jackrabbit.core.state.StateChangeDispatcher.notifyNodeRemoved(StateChangeDispatcher.java:207)
at org.apache.jackrabbit.core.state.SessionItemStateManager.nodeRemoved(SessionItemStateManager.java:983)
at org.apache.jackrabbit.core.state.NodeState.notifyNodeRemoved(NodeState.java:901)
at org.apache.jackrabbit.core.state.NodeState.removeChildNodeEntry(NodeState.java:388)- locked <0xffffffff502af0f0> (a org.apache.jackrabbit.core.state.NodeState)
at org.apache.jackrabbit.core.NodeImpl.removeChildNode(NodeImpl.java:615)
at org.apache.jackrabbit.core.ItemImpl.internalRemove(ItemImpl.java:847)
at org.apache.jackrabbit.core.ItemImpl.remove(ItemImpl.java:919)
at com.liferay.documentlibrary.util.JCRHook.deleteFile(JCRHook.java:349)
at com.liferay.documentlibrary.util.HookProxyImpl.deleteFile(HookProxyImpl.java:101)
at com.liferay.documentlibrary.util.SafeFileNameHookWrapper.deleteFile(SafeFileNameHookWrapper.java:146)
at com.liferay.documentlibrary.service.impl.DLServiceImpl.deleteFile(DLServiceImpl.java:105)
at sun.reflect.GeneratedMethodAccessor2144.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:309)
at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:110)
at com.liferay.portal.dao.jdbc.aop.DynamicDataSourceTransactionInterceptor.invoke(DynamicDataSourceTransactionInterceptor.java:44)
at com.liferay.portal.spring.aop.ChainableMethodAdvice.invoke(ChainableMethodAdvice.java:58)
at com.liferay.portal.spring.aop.ChainableMethodAdvice.invoke(ChainableMethodAdvice.java:58)