Random IndexNotFoundException when executing reindex after upgrade

• Type: Bug
• Status: Closed
• Resolution: Inactive
• Affects Version/s: Master
• Fix Version/s: 7.4.3.12 CE GA12, 7.4.13 DXP U8, Master
• Component/s: Search Infrastructure
• Labels:
  • QA-AUTOMATION
  • flaky_test_search
  • reviewed-by-pm
• Fix Priority: 3
• QA Test Name:
  • LocalFile.AssetsSharingUpgrade#ViewSharedAssetsArchive72101
• QA Test Score: 5

Steps to Reproduce:

1. Upgrade to master from previous liferay version
   • Set "indexReadOnly=true" during upgrade (osgi/configs/com.liferay.portal.search.configuration.IndexStatusManagerConfiguration.cfg)
   • Set "indexReadOnly=false" after upgrade
2. Start portal after upgrade
3. Navigate to Search admin portlet
4. Reindex all search indexes

Expected Result:
Reindex is always successful

Actual Result:
Reindex sometimes fails:

04:26:35,249 INFO  [liferay/background_task-4][SearchEngineInitializer:88] Reindexing Lucene started
04:26:47,760 INFO  [liferay/background_task-4][CompanyIndexFactory:51] 
04:27:17,774 INFO  [liferay/background_task-4][CompanyIndexFactory:51] liferay-0
04:27:18,261 WARN  [liferay/background_task-4][ProxyMessageListener:88] java.lang.RuntimeException: org.elasticsearch.index.IndexNotFoundException: [liferay-0] IndexNotFoundException[no such index]
java.lang.RuntimeException: org.elasticsearch.index.IndexNotFoundException: [liferay-0] IndexNotFoundException[no such index]
	at org.elasticsearch.cluster.metadata.IndexNameExpressionResolver$WildcardExpressionResolver.indexNotFoundException(IndexNameExpressionResolver.java:720)
	at org.elasticsearch.cluster.metadata.IndexNameExpressionResolver$WildcardExpressionResolver.innerResolve(IndexNameExpressionResolver.java:672)
	at org.elasticsearch.cluster.metadata.IndexNameExpressionResolver$WildcardExpressionResolver.resolve(IndexNameExpressionResolver.java:628)
	at org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.concreteIndices(IndexNameExpressionResolver.java:164)
	at org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.concreteIndices(IndexNameExpressionResolver.java:139)
	at org.elasticsearch.action.search.TransportSearchAction.resolveLocalIndices(TransportSearchAction.java:287)
	at org.elasticsearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:301)
	at org.elasticsearch.action.search.TransportSearchAction.lambda$doExecute$4(TransportSearchAction.java:193)
	at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:60)
	at org.elasticsearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:114)
	at org.elasticsearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:87)
	at org.elasticsearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:215)
	at org.elasticsearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:68)
	at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:167)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:139)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:81)
	at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:87)
	at org.elasticsearch.client.node.NodeClient.doExecute(NodeClient.java:76)
	at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:395)
	at org.elasticsearch.client.FilterClient.doExecute(FilterClient.java:67)
	at org.elasticsearch.client.ParentTaskAssigningClient.doExecute(ParentTaskAssigningClient.java:66)
	at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:395)
	at org.elasticsearch.client.support.AbstractClient.search(AbstractClient.java:522)
	at org.elasticsearch.index.reindex.ClientScrollableHitSource.lambda$doStart$0(ClientScrollableHitSource.java:78)
	at org.elasticsearch.index.reindex.ClientScrollableHitSource$1RetryHelper.doRun(ClientScrollableHitSource.java:143)
	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
	at org.elasticsearch.index.reindex.ClientScrollableHitSource.searchWithRetry(ClientScrollableHitSource.java:174)
	at org.elasticsearch.index.reindex.ClientScrollableHitSource.doStart(ClientScrollableHitSource.java:78)
	at org.elasticsearch.index.reindex.ScrollableHitSource.start(ScrollableHitSource.java:69)
	at org.elasticsearch.index.reindex.AbstractAsyncBulkByScrollAction.start(AbstractAsyncBulkByScrollAction.java:241)
	at org.elasticsearch.index.reindex.TransportDeleteByQueryAction.lambda$doExecute$0(TransportDeleteByQueryAction.java:62)
	at org.elasticsearch.index.reindex.BulkByScrollParallelizationHelper.sliceConditionally(BulkByScrollParallelizationHelper.java:104)
	at org.elasticsearch.index.reindex.BulkByScrollParallelizationHelper.startSlicedAction(BulkByScrollParallelizationHelper.java:81)
	at org.elasticsearch.index.reindex.TransportDeleteByQueryAction.doExecute(TransportDeleteByQueryAction.java:55)
	at org.elasticsearch.index.reindex.TransportDeleteByQueryAction.doExecute(TransportDeleteByQueryAction.java:37)
	at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:167)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:139)
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:81)
	at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:87)
	at org.elasticsearch.client.node.NodeClient.doExecute(NodeClient.java:76)
	at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:395)
	at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:384)
	at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:46)
	at org.elasticsearch.action.ActionRequestBuilder.get(ActionRequestBuilder.java:53)
	at com.liferay.portal.search.elasticsearch6.internal.search.engine.adapter.document.DeleteByQueryDocumentRequestExecutorImpl.execute(DeleteByQueryDocumentRequestExecutorImpl.java:48)
	at com.liferay.portal.search.elasticsearch6.internal.search.engine.adapter.document.ElasticsearchDocumentRequestExecutor.executeDocumentRequest(ElasticsearchDocumentRequestExecutor.java:57)
	at com.liferay.portal.search.engine.adapter.document.DeleteByQueryDocumentRequest.accept(DeleteByQueryDocumentRequest.java:36)
	at com.liferay.portal.search.engine.adapter.document.DeleteByQueryDocumentRequest.accept(DeleteByQueryDocumentRequest.java:23)
	at com.liferay.portal.search.elasticsearch6.internal.search.engine.adapter.ElasticsearchSearchEngineAdapterImpl.execute(ElasticsearchSearchEngineAdapterImpl.java:75)
	at com.liferay.portal.search.elasticsearch6.internal.ElasticsearchIndexWriter.deleteEntityDocuments(ElasticsearchIndexWriter.java:244)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.liferay.portal.kernel.messaging.proxy.ProxyRequest.execute(ProxyRequest.java:82)
	at com.liferay.portal.kernel.messaging.proxy.ProxyMessageListener.receive(ProxyMessageListener.java:58)
	at com.liferay.portal.kernel.messaging.InvokerMessageListener.receive(InvokerMessageListener.java:74)
	at com.liferay.portal.messaging.internal.SynchronousDestination.send(SynchronousDestination.java:46)
	at com.liferay.portal.messaging.internal.DefaultMessageBus.sendMessage(DefaultMessageBus.java:237)
	at com.liferay.portal.kernel.messaging.MessageBusUtil.sendMessage(MessageBusUtil.java:81)
	at com.liferay.portal.kernel.messaging.proxy.BaseMultiDestinationProxyBean.send(BaseMultiDestinationProxyBean.java:34)
	at com.liferay.portal.messaging.proxy.MultiDestinationMessagingProxyInvocationHandler.invoke(MultiDestinationMessagingProxyInvocationHandler.java:53)
	at com.sun.proxy.$Proxy235.deleteEntityDocuments(Unknown Source)
	at com.liferay.portal.search.internal.IndexWriterHelperImpl.deleteEntityDocuments(IndexWriterHelperImpl.java:240)
	at com.liferay.portal.kernel.search.IndexWriterHelperUtil.deleteEntityDocuments(IndexWriterHelperUtil.java:82)
	at com.liferay.portal.search.internal.SearchEngineInitializer.doReIndex(SearchEngineInitializer.java:134)
	at com.liferay.portal.search.internal.SearchEngineInitializer.reindex(SearchEngineInitializer.java:74)
	at com.liferay.portal.search.internal.SearchEngineInitializer.reindex(SearchEngineInitializer.java:70)
	at com.liferay.portal.search.internal.background.task.ReindexPortalBackgroundTaskExecutor.reindex(ReindexPortalBackgroundTaskExecutor.java:61)
	at com.liferay.portal.search.internal.background.task.ReindexBackgroundTaskExecutor.execute(ReindexBackgroundTaskExecutor.java:54)
	at com.liferay.portal.background.task.internal.SerialBackgroundTaskExecutor.execute(SerialBackgroundTaskExecutor.java:62)
	at com.liferay.portal.kernel.backgroundtask.DelegatingBackgroundTaskExecutor.execute(DelegatingBackgroundTaskExecutor.java:41)
	at com.liferay.portal.background.task.internal.ThreadLocalAwareBackgroundTaskExecutor.execute(ThreadLocalAwareBackgroundTaskExecutor.java:72)
	at com.liferay.portal.background.task.internal.messaging.BackgroundTaskMessageListener.doReceive(BackgroundTaskMessageListener.java:136)
	at com.liferay.portal.kernel.messaging.BaseMessageListener.receive(BaseMessageListener.java:26)
	at com.liferay.portal.kernel.messaging.InvokerMessageListener.receive(InvokerMessageListener.java:74)
	at com.liferay.portal.messaging.internal.ParallelDestination$1.run(ParallelDestination.java:56)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)
04:27:18,265 INFO  [default-24][SearchEngineInitializer:191] Reindexing of com.liferay.configuration.admin.web.internal.model.ConfigurationModel entities started
04:27:42,127 ERROR [default-24][BulkDocumentRequestExecutorImpl:56] failure in bulk execution:
[0]: index [liferay-0], type [LiferayDocumentType], id [com_liferay_configuration_admin_web_portlet_SystemSettingsPortlet_PORTLET_com.liferay.site.navigation.menu.web.internal.configuration.SiteNavigationMenuWebTemplateConfiguration], message [[liferay-0] IndexNotFoundException[no such index]]
[1]: index [liferay-0], type [LiferayDocumentType], id [com_liferay_configuration_admin_web_portlet_SystemSettingsPortlet_PORTLET_com.liferay.site.navigation.menu.web.internal.configuration.SiteNavigationMenuWebTemplateConfiguration], message [[liferay-0] IndexNotFoundException[no such index]]
04:27:42,128 WARN  [default-24][ProxyMessageListener:88] com.liferay.portal.kernel.exception.SystemException: Update failed
com.liferay.portal.kernel.exception.SystemException: Update failed
	at com.liferay.portal.search.elasticsearch6.internal.ElasticsearchIndexWriter.updateDocument(ElasticsearchIndexWriter.java:367)
	at sun.reflect.GeneratedMethodAccessor221.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.liferay.portal.kernel.messaging.proxy.ProxyRequest.execute(ProxyRequest.java:82)
	at com.liferay.portal.kernel.messaging.proxy.ProxyMessageListener.receive(ProxyMessageListener.java:58)
	at com.liferay.portal.kernel.messaging.InvokerMessageListener.receive(InvokerMessageListener.java:74)
	at com.liferay.portal.messaging.internal.SynchronousDestination.send(SynchronousDestination.java:46)
	at com.liferay.portal.messaging.internal.DefaultMessageBus.sendMessage(DefaultMessageBus.java:237)
	at com.liferay.portal.kernel.messaging.MessageBusUtil.sendMessage(MessageBusUtil.java:81)
	at com.liferay.portal.kernel.messaging.proxy.BaseMultiDestinationProxyBean.send(BaseMultiDestinationProxyBean.java:34)
	at com.liferay.portal.messaging.proxy.MultiDestinationMessagingProxyInvocationHandler.invoke(MultiDestinationMessagingProxyInvocationHandler.java:53)
	at com.sun.proxy.$Proxy235.updateDocument(Unknown Source)
	at com.liferay.portal.search.internal.IndexWriterHelperImpl.updateDocument(IndexWriterHelperImpl.java:604)
	at com.liferay.configuration.admin.web.internal.search.ConfigurationModelIndexer.doReindex(ConfigurationModelIndexer.java:363)
	at com.liferay.configuration.admin.web.internal.search.ConfigurationModelIndexer.doReindex(ConfigurationModelIndexer.java:381)
	at com.liferay.portal.kernel.search.BaseIndexer.reindex(BaseIndexer.java:506)
	at com.liferay.portal.search.internal.SearchEngineInitializer.reindex(SearchEngineInitializer.java:196)
	at com.liferay.portal.search.internal.SearchEngineInitializer$1.call(SearchEngineInitializer.java:149)
	at com.liferay.portal.search.internal.SearchEngineInitializer$1.call(SearchEngineInitializer.java:140)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)

Liferay log: https://testray.liferay.com/reports/production/logs/test-1-16/1585278902095/test-portal-acceptance-pullrequest%28master%29/5702/functional-upgrade-tomcat90-mysql57-jdk8_stable/1/0/liferay-log.txt.gz
Poshi log: https://test-1-10.liferay.com/job/test-portal-acceptance-pullrequest-batch(master)/306228/AXIS_VARIABLE=0,label_exp=!master/consoleText

Poshi Report: https://testray.liferay.com/reports/production/logs/test-1-16/1585278902095/test-portal-acceptance-pullrequest(master)/5702/functional-upgrade-tomcat90-mysql57-jdk8_stable/1/0/LocalFile.PortalSmokeUpgrade_ViewPortalSmokeArchive71103/index.html.gz

Reproduced on:
Master: 7bc421b7a444cce6cf83da9e551636bdbecac744