Intermittent SocketTimeoutException occurring on Elasticsearch 7 unit tests

• Type: Regression Bug
• Status: Closed
• Resolution: Inactive
• Affects Version/s: 7.2.X, Master
• Fix Version/s: None
• Component/s: Search Infrastructure
• Labels:
  • QA-AUTOMATION
  • backend_test_fix
  • flaky_test_search
• Fix Priority: 3
• QA Test Name:
  • LocalFile.SAML#ViewSingleLogoutAndForceAuthWithMultipleSPs
• QA Test Score: 4

Incidences:

Date	Test Name	Location	Full Console
3/9/2020	elasticsearch7.internal.aggregation.bucket	BChan PR	link
3/9/2020	ElasticsearchSearchEngineAdapterIndexRequestTest	BChan PR	link
3/11/2020	com.liferay.portal.search.elasticsearch7.internal	Dev PR	link
3/18/2020	AvgAggregationTest#testAvgAggregation	Dev Acceptance	link
3/25/2020	CommonSearchSourceBuilderAssemblerImplTest.testPartsWillNarrowDownWhatMainQueryFinds	master dev PR	link
3/31/2020	ExtendedStatsBucketPipelineAggregationTest.testStatsBucketPipeline	ci:test:search-es7	link
4/3/2020	HistogramAggregationTest.testDateHistogramAggregationIntervalDay	ci:test:search-es7	link
4/14/2020	WeightedAvgAggregationTest.testWeightedAverage	ci:test:search-es7	link
4/14/2020	AssetTagNamesFieldQueryBuilderTest.testMultiwordPhrasePrefixesElasticsearch	ci:test:search-es7	link
4/16/2020	ElasticsearchIndexSearcherExceptionsTest.testExceptionThrownWhenQueryMalformedSearchCount ElasticsearchIndexSearcherLogExceptionsOnlyTest.testExceptionOnlyLoggedWhenQueryMalformedSearchCount ElasticsearchIndexSearcherLoggingTest.testCountSearchRequestExecutorLogsViaIndexer	Dev PR	link
4/20/2020	WrapperQueryTest.testWrapperQuery	ci:test:search-es7	link
4/29/2020	ExtendedStatsBucketPipelineAggregationTest.testStatsBucketPipeline MovingFunctionPipelineAggregationTest.testMovingFunction StringQueryTest.testBooleanOperatorNotDeepElasticsearch	ci:test:search-es7	link
5/4/2020	TermQueryTest.testTermQuery	Dev PR	link
5/5/2020	ExistsQueryTest.testExistsQuery	ci:test:search-es7	link
5/11/2020	ElasticsearchSearchEngineTest.testInitializeAfterReconnect ElasticsearchSearchEngineTest.testRestore ElasticsearchSearchEngineTest.testBackup	ci:test:search-es7	link
5/12/2020	StringQueryTest.testBooleanOperatorNotDeepElasticsearch	ci:test:search-es7	link
5/16/2020	PercentileRanksAggregationTest#testPercentileRanksAggregationTDigest	Master CI Analysis	link
7/20/2020	GeoDistanceFilterTest.testGeoDistanceRangeFilter	ci:test:search	link

Example stacktrace:

java.lang.RuntimeException: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-0 [ACTIVE]
	at com.liferay.portal.search.elasticsearch7.internal.connection.IndexCreator.createIndex(IndexCreator.java:69)
	at com.liferay.portal.search.elasticsearch7.internal.ElasticsearchIndexingFixture.createIndex(ElasticsearchIndexingFixture.java:239)
	at com.liferay.portal.search.elasticsearch7.internal.ElasticsearchIndexingFixture.setUp(ElasticsearchIndexingFixture.java:121)
	at com.liferay.portal.search.test.util.indexing.BaseIndexingTestCase.setUpIndexingFixture(BaseIndexingTestCase.java:287)
	at com.liferay.portal.search.test.util.indexing.BaseIndexingTestCase.setUp(BaseIndexingTestCase.java:110)
	at com.liferay.portal.search.test.util.document.BaseDocumentTestCase.setUp(BaseDocumentTestCase.java:36)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24)
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
	at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
	at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
	at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
	at org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.runTestClass(JUnitTestClassExecutor.java:110)
	at org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.execute(JUnitTestClassExecutor.java:58)
	at org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.execute(JUnitTestClassExecutor.java:38)
	at org.gradle.api.internal.tasks.testing.junit.AbstractJUnitTestClassProcessor.processTestClass(AbstractJUnitTestClassProcessor.java:62)
	at org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.processTestClass(SuiteTestClassProcessor.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36)
	at org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
	at org.gradle.internal.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:33)
	at org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:94)
	at com.sun.proxy.$Proxy2.processTestClass(Unknown Source)
	at org.gradle.api.internal.tasks.testing.worker.TestWorker.processTestClass(TestWorker.java:118)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36)
	at org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
	at org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:182)
	at org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:164)
	at org.gradle.internal.remote.internal.hub.MessageHub$Handler.run(MessageHub.java:412)
	at org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:64)
	at org.gradle.internal.concurrent.ManagedExecutorImpl$1.run(ManagedExecutorImpl.java:48)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at org.gradle.internal.concurrent.ThreadFactoryImpl$ManagedThreadRunnable.run(ThreadFactoryImpl.java:56)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-0 [ACTIVE]
	at org.elasticsearch.client.RestClient.extractAndWrapCause(RestClient.java:773)
	at org.elasticsearch.client.RestClient.performRequest(RestClient.java:218)
	at org.elasticsearch.client.RestClient.performRequest(RestClient.java:205)
	at org.elasticsearch.client.RestHighLevelClient.internalPerformRequest(RestHighLevelClient.java:1454)
	at org.elasticsearch.client.RestHighLevelClient.performRequest(RestHighLevelClient.java:1424)
	at org.elasticsearch.client.RestHighLevelClient.performRequestAndParseEntity(RestHighLevelClient.java:1394)
	at org.elasticsearch.client.IndicesClient.create(IndicesClient.java:163)
	at com.liferay.portal.search.elasticsearch7.internal.connection.IndexCreator.createIndex(IndexCreator.java:66)
	... 57 more
Caused by: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-0 [ACTIVE]
	at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.timeout(HttpAsyncRequestExecutor.java:381)
	at org.apache.http.impl.nio.client.InternalIODispatch.onTimeout(InternalIODispatch.java:92)
	at org.apache.http.impl.nio.client.InternalIODispatch.onTimeout(InternalIODispatch.java:39)
	at org.apache.http.impl.nio.reactor.AbstractIODispatch.timeout(AbstractIODispatch.java:175)
	at org.apache.http.impl.nio.reactor.BaseIOReactor.sessionTimedOut(BaseIOReactor.java:263)
	at org.apache.http.impl.nio.reactor.AbstractIOReactor.timeoutCheck(AbstractIOReactor.java:492)
	at org.apache.http.impl.nio.reactor.BaseIOReactor.validate(BaseIOReactor.java:213)
	at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:280)
	at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104)
	at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:588)
	... 1 more