Uploaded image for project: 'PUBLIC - Liferay Portal Community Edition'
  1. PUBLIC - Liferay Portal Community Edition
  2. LPS-30860

Consolidate "high performance" dynamic query pagination pattern where you query lots of data and performing an action on each item

    Details

      Description

      LPS-26223 introduced a pattern for paginating large sets of data. Imagine if you wanted to do something to 1 million users. Grabbing them all into memory and performing an action on each user would kill the database.

      Very old pattern:

      protected void reindexUsers(long companyId) throws Exception {
      	int count = UserLocalServiceUtil.getCompanyUsersCount(companyId);
      
      	int pages = count / UserIndexer.DEFAULT_INTERVAL;
      
      	for (int i = 0; i <= pages; i++) {
      		int start = (i * UserIndexer.DEFAULT_INTERVAL);
      		int end = start + UserIndexer.DEFAULT_INTERVAL;
      
      		reindexUsers(companyId, start, end);
      	}
      }
      
      protected void reindexUsers(long companyId, int start, int end)
      	throws Exception {
      
      	List<User> users = UserLocalServiceUtil.getCompanyUsers(
      		companyId, start, end);
      

      LPS-26223 fixed that by introducing this pattern:

      protected void reindexUsers(long companyId) throws Exception {
      	DynamicQuery dynamicQuery = DynamicQueryFactoryUtil.forClass(
      		User.class, PortalClassLoaderUtil.getClassLoader());
      
      	Projection minUserIdProjection = ProjectionFactoryUtil.min("userId");
      	Projection maxUserIdProjection = ProjectionFactoryUtil.max("userId");
      
      	ProjectionList projectionList = ProjectionFactoryUtil.projectionList();
      
      	projectionList.add(minUserIdProjection);
      	projectionList.add(maxUserIdProjection);
      
      	dynamicQuery.setProjection(projectionList);
      
      	addReindexCriteria(dynamicQuery, companyId);
      
      	List<Object[]> results = UserLocalServiceUtil.dynamicQuery(
      		dynamicQuery);
      
      	Object[] minAndMaxUserIds = results.get(0);
      
      	if ((minAndMaxUserIds[0] == null) || (minAndMaxUserIds[1] == null)) {
      		return;
      	}
      
      	long minUserId = (Long)minAndMaxUserIds[0];
      	long maxUserId = (Long)minAndMaxUserIds[1];
      
      	long startUserId = minUserId;
      	long endUserId = startUserId + DEFAULT_INTERVAL;
      
      	while (startUserId <= maxUserId) {
      		reindexUsers(companyId, startUserId, endUserId);
      
      		startUserId = endUserId;
      		endUserId += DEFAULT_INTERVAL;
      	}
      }
      
      protected void reindexUsers(
      		long companyId, long startUserId, long endUserId)
      	throws Exception {
      
      	DynamicQuery dynamicQuery = DynamicQueryFactoryUtil.forClass(
      		User.class, PortalClassLoaderUtil.getClassLoader());
      
      	Property property = PropertyFactoryUtil.forName("userId");
      
      	dynamicQuery.add(property.ge(startUserId));
      	dynamicQuery.add(property.lt(endUserId));
      
      	addReindexCriteria(dynamicQuery, companyId);
      
      	List<User> users = UserLocalServiceUtil.dynamicQuery(dynamicQuery);
      
      

      But it's still way too verbose, so this ticket introduces a pattern based on a new class called "ActionableDynamicQuery"

      protected void reindexUsers(long companyId)
      	throws PortalException, SystemException {
      
      	final Collection<Document> documents = new ArrayList<Document>();
      
      	ActionableDynamicQuery actionableDynamicQuery =
      		new UserActionableDynamicQuery() {
      
      		@Override
      		protected void performAction(Object object) throws PortalException {
      			User user = (User)object;
      
      			if (!user.isDefaultUser()) {
      				Document document = getDocument(user);
      
      				documents.add(document);
      			}
      		}
      
      	};
      
      	actionableDynamicQuery.setCompanyId(companyId);
      
      	actionableDynamicQuery.performActions();
      
      	SearchEngineUtil.updateDocuments(
      		getSearchEngineId(), companyId, documents);
      }
      

      Much leaner now, another example for Blogs is:

      protected void reindexEntries(long companyId)
      	throws PortalException, SystemException {
      
      	final Collection<Document> documents = new ArrayList<Document>();
      
      	ActionableDynamicQuery actionableDynamicQuery =
      		new BaseActionableDynamicQuery() {
      
      		@Override
      		protected void addCriteria(DynamicQuery dynamicQuery) {
      			Property displayDateProperty = PropertyFactoryUtil.forName(
      				"displayDate");
      
      			dynamicQuery.add(displayDateProperty.lt(new Date()));
      
      			Property statusProperty = PropertyFactoryUtil.forName("status");
      
      			Integer[] statuses = {
      				WorkflowConstants.STATUS_APPROVED,
      				WorkflowConstants.STATUS_IN_TRASH
      			};
      
      			dynamicQuery.add(statusProperty.in(statuses));
      		}
      
      		@Override
      		protected void performAction(Object object) throws PortalException {
      			BlogsEntry entry = (BlogsEntry)object;
      
      			Document document = getDocument(entry);
      
      			documents.add(document);
      		}
      
      	};
      
      	actionableDynamicQuery.setBaseLocalService(
      		BlogsEntryLocalServiceUtil.getService());
      	actionableDynamicQuery.setClass(BlogsEntry.class);
      	actionableDynamicQuery.setClassLoader(
      		PACLClassLoaderUtil.getPortalClassLoader());
      	actionableDynamicQuery.setCompanyId(companyId);
      	actionableDynamicQuery.setPrimaryKeyPropertyName("entryId");
      
      	actionableDynamicQuery.performActions();
      
      	SearchEngineUtil.updateDocuments(
      		getSearchEngineId(), companyId, documents);
      }
      

        Attachments

          Issue Links

            Activity

              People

              • Votes:
                0 Vote for this issue
                Watchers:
                2 Start watching this issue

                Dates

                • Created:
                  Updated:
                  Resolved:

                  Subcomponents