Uploaded image for project: 'PUBLIC - Liferay Commerce'
  1. PUBLIC - Liferay Commerce
  2. COMMERCE-6835

Batch export fails when exporting 100_000+ entities

    Details

    • Type: Bug
    • Status: Selected for Development
    • Priority: Major
    • Resolution: Unresolved
    • Affects Version/s: Master
    • Fix Version/s: None
    • Component/s: Data Integration
    • Labels:
      None
    • Severity of Issue:
      Major

      Description

      When exporting 100_000 products, Batch export task fails with error:

      2021-06-30 08:36:25.866 ERROR [default-4][ElasticsearchIndexSearcher:165] java.lang.RuntimeException: org.elasticsearch.ElasticsearchStatusException: ElasticsearchStatusException[Elasticsearch exception [type=search_phase_execution_exception, reason=all shards failed]]; nested: ElasticsearchException[Elasticsearch exception [type=illegal_argument_exception, reason=Result window is too large, from + size must be less than or equal to: [10000] but was [10100]. See the scroll api for a more efficient way to request large data sets. This limit can be set by changing the [index.max_result_window] index level setting.]]; nested: ElasticsearchException[Elasticsearch exception [type=illegal_argument_exception, reason=Result window is too large, from + size must be less than or equal to: [10000] but was [10100]. See the scroll api for a more efficient way to request large data sets. This limit can be set by changing the [index.max_result_window] index level setting.]];java.lang.RuntimeException: org.elasticsearch.ElasticsearchStatusException: ElasticsearchStatusException[Elasticsearch exception [type=search_phase_execution_exception, reason=all shards failed]]; nested: ElasticsearchException[Elasticsearch exception [type=illegal_argument_exception, reason=Result window is too large, from + size must be less than or equal to: [10000] but was [10100]. See the scroll api for a more efficient way to request large data sets. This limit can be set by changing the [index.max_result_window] index level setting.]]; nested: ElasticsearchException[Elasticsearch exception [type=illegal_argument_exception, reason=Result window is too large, from + size must be less than or equal to: [10000] but was [10100]. See the scroll api for a more efficient way to request large data sets. This limit can be set by changing the [index.max_result_window] index level setting.]]; at org.elasticsearch.rest.BytesRestResponse.errorFromXContent(BytesRestResponse.java:187) ~[elasticsearch-7.9.0.jar:7.9.0] at org.elasticsearch.client.RestHighLevelClient.parseEntity(RestHighLevelClient.java:1892) ~[elasticsearch-rest-high-level-client-7.9.0.jar:7.9.0] at org.elasticsearch.client.RestHighLevelClient.parseResponseException(RestHighLevelClient.java:1869) ~[elasticsearch-rest-high-level-client-7.9.0.jar:7.9.0] at org.elasticsearch.client.RestHighLevelClient.internalPerformRequest(RestHighLevelClient.java:1626) ~[elasticsearch-rest-high-level-client-7.9.0.jar:7.9.0] at org.elasticsearch.client.RestHighLevelClient.performRequest(RestHighLevelClient.java:1583) ~[elasticsearch-rest-high-level-client-7.9.0.jar:7.9.0] at org.elasticsearch.client.RestHighLevelClient.performRequestAndParseEntity(RestHighLevelClient.java:1553) ~[elasticsearch-rest-high-level-client-7.9.0.jar:7.9.0] at org.elasticsearch.client.RestHighLevelClient.search(RestHighLevelClient.java:1069) ~[elasticsearch-rest-high-level-client-7.9.0.jar:7.9.0] at com.liferay.portal.search.elasticsearch7.internal.search.engine.adapter.search.SearchSearchRequestExecutorImpl.getSearchResponse(SearchSearchRequestExecutorImpl.java:95) ~[bundleFile:?] at com.liferay.portal.search.elasticsearch7.internal.search.engine.adapter.search.SearchSearchRequestExecutorImpl.execute(SearchSearchRequestExecutorImpl.java:66) ~[bundleFile:?] at com.liferay.portal.search.elasticsearch7.internal.search.engine.adapter.search.ElasticsearchSearchRequestExecutor.executeSearchRequest(ElasticsearchSearchRequestExecutor.java:59) ~[bundleFile:?] at com.liferay.portal.search.engine.adapter.search.SearchSearchRequest.accept(SearchSearchRequest.java:45) ~[bundleFile:?] at com.liferay.portal.search.engine.adapter.search.SearchSearchRequest.accept(SearchSearchRequest.java:34) ~[bundleFile:?] at com.liferay.portal.search.elasticsearch7.internal.search.engine.adapter.ElasticsearchSearchEngineAdapterImpl.execute(ElasticsearchSearchEngineAdapterImpl.java:103) ~[bundleFile:?] at com.liferay.portal.search.elasticsearch7.internal.ElasticsearchIndexSearcher.search(ElasticsearchIndexSearcher.java:127) ~[bundleFile:?] at sun.reflect.GeneratedMethodAccessor806.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_212] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_212] at com.liferay.portal.kernel.messaging.proxy.ProxyRequest.execute(ProxyRequest.java:82) ~[portal-kernel.jar:?] at com.liferay.portal.kernel.messaging.proxy.ProxyMessageListener.receive(ProxyMessageListener.java:58) ~[portal-kernel.jar:?] at com.liferay.portal.kernel.messaging.InvokerMessageListener.receive(InvokerMessageListener.java:74) ~[portal-kernel.jar:?] at com.liferay.portal.messaging.internal.SynchronousDestination.send(SynchronousDestination.java:46) ~[?:?] at com.liferay.portal.messaging.internal.sender.DirectSynchronousMessageSender.send(DirectSynchronousMessageSender.java:59) ~[?:?] at com.liferay.portal.kernel.messaging.proxy.BaseMultiDestinationProxyBean.synchronousSend(BaseMultiDestinationProxyBean.java:53) ~[portal-kernel.jar:?] at com.liferay.portal.messaging.proxy.MultiDestinationMessagingProxyInvocationHandler.invoke(MultiDestinationMessagingProxyInvocationHandler.java:58) ~[portal-impl.jar:?] at com.sun.proxy.$Proxy260.search(Unknown Source) ~[?:?] at com.liferay.portal.search.internal.IndexSearcherHelperImpl.search(IndexSearcherHelperImpl.java:70) ~[?:?] at com.liferay.portal.kernel.search.IndexSearcherHelperUtil.search(IndexSearcherHelperUtil.java:36) ~[portal-kernel.jar:?] at com.liferay.portal.kernel.search.BaseIndexer.doSearch(BaseIndexer.java:1192) ~[portal-kernel.jar:?] at com.liferay.portal.kernel.search.BaseIndexer._search(BaseIndexer.java:1591) ~[portal-kernel.jar:?] at com.liferay.portal.kernel.search.BaseIndexer.search(BaseIndexer.java:567) ~[portal-kernel.jar:?] at com.liferay.portal.vulcan.util.SearchUtil.search(SearchUtil.java:123) ~[?:?] at com.liferay.headless.commerce.admin.catalog.internal.helper.v1_0.ProductHelper.getProductsPage(ProductHelper.java:54) ~[?:?] at com.liferay.headless.commerce.admin.catalog.internal.resource.v1_0.ProductResourceImpl.getProductsPage(ProductResourceImpl.java:227) ~[?:?] at com.liferay.headless.commerce.admin.catalog.internal.resource.v1_0.BaseProductResourceImpl.read(BaseProductResourceImpl.java:439) ~[?:?] at com.liferay.portal.vulcan.internal.batch.engine.VulcanBatchEngineTaskItemDelegateAdaptor.read(VulcanBatchEngineTaskItemDelegateAdaptor.java:108) ~[?:?] at com.liferay.batch.engine.internal.item.BatchEngineTaskItemDelegateExecutor.getItems(BatchEngineTaskItemDelegateExecutor.java:77) ~[?:?] at com.liferay.batch.engine.internal.BatchEngineExportTaskExecutorImpl._exportItems(BatchEngineExportTaskExecutorImpl.java:155) ~[?:?] at com.liferay.batch.engine.internal.BatchEngineExportTaskExecutorImpl.lambda$execute$0(BatchEngineExportTaskExecutorImpl.java:78) ~[?:?] at com.liferay.batch.engine.internal.BatchEngineTaskExecutorUtil.execute(BatchEngineTaskExecutorUtil.java:44) ~[?:?] at com.liferay.batch.engine.internal.BatchEngineExportTaskExecutorImpl.execute(BatchEngineExportTaskExecutorImpl.java:77) ~[?:?] at com.liferay.headless.batch.engine.internal.resource.v1_0.ExportTaskResourceImpl.lambda$postExportTask$1(ExportTaskResourceImpl.java:119) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [?:1.8.0_212] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_212] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_212] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_212] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_212] Suppressed: org.elasticsearch.client.ResponseException: method [POST], host [http://127.0.0.1:9201], URI [/liferay-37609/_search?typed_keys=true&max_concurrent_shard_requests=5&ignore_unavailable=false&expand_wildcards=open&allow_no_indices=true&ignore_throttled=true&search_type=query_then_fetch&batched_reduce_size=512&ccs_minimize_roundtrips=true], status line [HTTP/1.1 400 Bad Request]{"error":{"root_cause":[{"type":"illegal_argument_exception","reason":"Result window is too large, from + size must be less than or equal to: [10000] but was [10100]. See the scroll api for a more efficient way to request large data sets. This limit can be set by changing the [index.max_result_window] index level setting."}],"type":"search_phase_execution_exception","reason":"all shards failed","phase":"query","grouped":true,"failed_shards":[{"shard":0,"index":"liferay-37609","node":"MQ3WLevET96gnHmmANFb-Q","reason":{"type":"illegal_argument_exception","reason":"Result window is too large, from + size must be less than or equal to: [10000] but was [10100]. See the scroll api for a more efficient way to request large data sets. This limit can be set by changing the [index.max_result_window] index level setting."}}],"caused_by":{"type":"illegal_argument_exception","reason":"Result window is too large, from + size must be less than or equal to: [10000] but was [10100]. See the scroll api for a more efficient way to request large data sets. This limit can be set by changing the [index.max_result_window] index level setting.","caused_by":{"type":"illegal_argument_exception","reason":"Result window is too large, from + size must be less than or equal to: [10000] but was [10100]. See the scroll api for a more efficient way to request large data sets. This limit can be set by changing the [index.max_result_window] index level setting."}}},"status":400} at org.elasticsearch.client.RestClient.convertResponse(RestClient.java:302) ~[elasticsearch-rest-client-7.9.0.jar:7.9.0] at org.elasticsearch.client.RestClient.performRequest(RestClient.java:272) ~[elasticsearch-rest-client-7.9.0.jar:7.9.0] at org.elasticsearch.client.RestClient.performRequest(RestClient.java:246) ~[elasticsearch-rest-client-7.9.0.jar:7.9.0] at org.elasticsearch.client.RestHighLevelClient.internalPerformRequest(RestHighLevelClient.java:1613) ~[elasticsearch-rest-high-level-client-7.9.0.jar:7.9.0] at org.elasticsearch.client.RestHighLevelClient.performRequest(RestHighLevelClient.java:1583) ~[elasticsearch-rest-high-level-client-7.9.0.jar:7.9.0] at org.elasticsearch.client.RestHighLevelClient.performRequestAndParseEntity(RestHighLevelClient.java:1553) ~[elasticsearch-rest-high-level-client-7.9.0.jar:7.9.0] at org.elasticsearch.client.RestHighLevelClient.search(RestHighLevelClient.java:1069) ~[elasticsearch-rest-high-level-client-7.9.0.jar:7.9.0] at com.liferay.portal.search.elasticsearch7.internal.search.engine.adapter.search.SearchSearchRequestExecutorImpl.getSearchResponse(SearchSearchRequestExecutorImpl.java:95) ~[bundleFile:?] at com.liferay.portal.search.elasticsearch7.internal.search.engine.adapter.search.SearchSearchRequestExecutorImpl.execute(SearchSearchRequestExecutorImpl.java:66) ~[bundleFile:?] at com.liferay.portal.search.elasticsearch7.internal.search.engine.adapter.search.ElasticsearchSearchRequestExecutor.executeSearchRequest(ElasticsearchSearchRequestExecutor.java:59) ~[bundleFile:?] at com.liferay.portal.search.engine.adapter.search.SearchSearchRequest.accept(SearchSearchRequest.java:45) ~[bundleFile:?] at com.liferay.portal.search.engine.adapter.search.SearchSearchRequest.accept(SearchSearchRequest.java:34) ~[bundleFile:?] at com.liferay.portal.search.elasticsearch7.internal.search.engine.adapter.ElasticsearchSearchEngineAdapterImpl.execute(ElasticsearchSearchEngineAdapterImpl.java:103) ~[bundleFile:?] at com.liferay.portal.search.elasticsearch7.internal.ElasticsearchIndexSearcher.search(ElasticsearchIndexSearcher.java:127) ~[bundleFile:?] at sun.reflect.GeneratedMethodAccessor806.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_212] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_212] at com.liferay.portal.kernel.messaging.proxy.ProxyRequest.execute(ProxyRequest.java:82) ~[portal-kernel.jar:?] at com.liferay.portal.kernel.messaging.proxy.ProxyMessageListener.receive(ProxyMessageListener.java:58) ~[portal-kernel.jar:?] at com.liferay.portal.kernel.messaging.InvokerMessageListener.receive(InvokerMessageListener.java:74) ~[portal-kernel.jar:?] at com.liferay.portal.messaging.internal.SynchronousDestination.send(SynchronousDestination.java:46) ~[?:?] at com.liferay.portal.messaging.internal.sender.DirectSynchronousMessageSender.send(DirectSynchronousMessageSender.java:59) ~[?:?] at com.liferay.portal.kernel.messaging.proxy.BaseMultiDestinationProxyBean.synchronousSend(BaseMultiDestinationProxyBean.java:53) ~[portal-kernel.jar:?] at com.liferay.portal.messaging.proxy.MultiDestinationMessagingProxyInvocationHandler.invoke(MultiDestinationMessagingProxyInvocationHandler.java:58) ~[portal-impl.jar:?] at com.sun.proxy.$Proxy260.search(Unknown Source) ~[?:?] at com.liferay.portal.search.internal.IndexSearcherHelperImpl.search(IndexSearcherHelperImpl.java:70) ~[?:?] at com.liferay.portal.kernel.search.IndexSearcherHelperUtil.search(IndexSearcherHelperUtil.java:36) ~[portal-kernel.jar:?] at com.liferay.portal.kernel.search.BaseIndexer.doSearch(BaseIndexer.java:1192) ~[portal-kernel.jar:?] at com.liferay.portal.kernel.search.BaseIndexer._search(BaseIndexer.java:1591) ~[portal-kernel.jar:?] at com.liferay.portal.kernel.search.BaseIndexer.search(BaseIndexer.java:567) ~[portal-kernel.jar:?] at com.liferay.portal.vulcan.util.SearchUtil.search(SearchUtil.java:123) ~[?:?] at com.liferay.headless.commerce.admin.catalog.internal.helper.v1_0.ProductHelper.getProductsPage(ProductHelper.java:54) ~[?:?] at com.liferay.headless.commerce.admin.catalog.internal.resource.v1_0.ProductResourceImpl.getProductsPage(ProductResourceImpl.java:227) ~[?:?] at com.liferay.headless.commerce.admin.catalog.internal.resource.v1_0.BaseProductResourceImpl.read(BaseProductResourceImpl.java:439) ~[?:?] at com.liferay.portal.vulcan.internal.batch.engine.VulcanBatchEngineTaskItemDelegateAdaptor.read(VulcanBatchEngineTaskItemDelegateAdaptor.java:108) ~[?:?] at com.liferay.batch.engine.internal.item.BatchEngineTaskItemDelegateExecutor.getItems(BatchEngineTaskItemDelegateExecutor.java:77) ~[?:?] at com.liferay.batch.engine.internal.BatchEngineExportTaskExecutorImpl._exportItems(BatchEngineExportTaskExecutorImpl.java:155) ~[?:?] at com.liferay.batch.engine.internal.BatchEngineExportTaskExecutorImpl.lambda$execute$0(BatchEngineExportTaskExecutorImpl.java:78) ~[?:?] at com.liferay.batch.engine.internal.BatchEngineTaskExecutorUtil.execute(BatchEngineTaskExecutorUtil.java:44) ~[?:?] at com.liferay.batch.engine.internal.BatchEngineExportTaskExecutorImpl.execute(BatchEngineExportTaskExecutorImpl.java:77) ~[?:?] at com.liferay.headless.batch.engine.internal.resource.v1_0.ExportTaskResourceImpl.lambda$postExportTask$1(ExportTaskResourceImpl.java:119) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [?:1.8.0_212] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_212] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_212] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_212] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_212]2021-06-30 08:36:25.932 ERROR [default-4][BatchEngineExportTaskExecutorImpl:86] Unable to update batch engine export task {mvccVersion=101, uuid=64759931-e893-5750-e01b-07a03be0286a, batchEngineExportTaskId=2001, companyId=37609, userId=37643, createDate=Wed Jun 30 08:35:19 GMT 2021, modifiedDate=Wed Jun 30 08:36:25 GMT 2021, callbackURL=, className=com.liferay.headless.commerce.admin.catalog.dto.v1_0.Product, contentType=JSON, endTime=null, errorMessage=, fieldNames=active,catalogId,name,productId,productType, executeStatus=STARTED, parameters={contentType=JSON, siteId=84114, className=com.liferay.headless.commerce.admin.catalog.dto.v1_0.Product}, processedItems=0, startTime=Wed Jun 30 08:35:19 GMT 2021, taskItemDelegateName=, totalItems=0}java.lang.NullPointerException: null at com.liferay.portal.vulcan.util.SearchUtil._getFacets(SearchUtil.java:318) ~[?:?] at com.liferay.portal.vulcan.util.SearchUtil.search(SearchUtil.java:139) ~[?:?] at com.liferay.headless.commerce.admin.catalog.internal.helper.v1_0.ProductHelper.getProductsPage(ProductHelper.java:54) ~[?:?] at com.liferay.headless.commerce.admin.catalog.internal.resource.v1_0.ProductResourceImpl.getProductsPage(ProductResourceImpl.java:227) ~[?:?] at com.liferay.headless.commerce.admin.catalog.internal.resource.v1_0.BaseProductResourceImpl.read(BaseProductResourceImpl.java:439) ~[?:?] at com.liferay.portal.vulcan.internal.batch.engine.VulcanBatchEngineTaskItemDelegateAdaptor.read(VulcanBatchEngineTaskItemDelegateAdaptor.java:108) ~[?:?] at com.liferay.batch.engine.internal.item.BatchEngineTaskItemDelegateExecutor.getItems(BatchEngineTaskItemDelegateExecutor.java:77) ~[?:?] at com.liferay.batch.engine.internal.BatchEngineExportTaskExecutorImpl._exportItems(BatchEngineExportTaskExecutorImpl.java:155) ~[?:?] at com.liferay.batch.engine.internal.BatchEngineExportTaskExecutorImpl.lambda$execute$0(BatchEngineExportTaskExecutorImpl.java:78) ~[?:?] at com.liferay.batch.engine.internal.BatchEngineTaskExecutorUtil.execute(BatchEngineTaskExecutorUtil.java:44) ~[?:?] at com.liferay.batch.engine.internal.BatchEngineExportTaskExecutorImpl.execute(BatchEngineExportTaskExecutorImpl.java:77) ~[?:?] at com.liferay.headless.batch.engine.internal.resource.v1_0.ExportTaskResourceImpl.lambda$postExportTask$1(ExportTaskResourceImpl.java:119) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [?:1.8.0_212] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_212] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_212] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_212] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_212]
      

        Attachments

          Issue Links

            Activity

              People

              Assignee:
              matija.petanjek Matija Petanjek
              Reporter:
              matija.petanjek Matija Petanjek
              Votes:
              0 Vote for this issue
              Watchers:
              0 Start watching this issue

                Dates

                Created:
                Updated:

                  Packages

                  Version Package