PUBLIC - Liferay Portal Community Edition
  1. PUBLIC - Liferay Portal Community Edition
  2. LPS-30860

Consolidate "high performance" dynamic query pagination pattern where you query lots of data and performing an action on each item

    Details

    • Similar Issues:
      Show 5 results 

      Description

      LPS-26223 introduced a pattern for paginating large sets of data. Imagine if you wanted to do something to 1 million users. Grabbing them all into memory and performing an action on each user would kill the database.

      Very old pattern:

      protected void reindexUsers(long companyId) throws Exception {
      	int count = UserLocalServiceUtil.getCompanyUsersCount(companyId);
      
      	int pages = count / UserIndexer.DEFAULT_INTERVAL;
      
      	for (int i = 0; i <= pages; i++) {
      		int start = (i * UserIndexer.DEFAULT_INTERVAL);
      		int end = start + UserIndexer.DEFAULT_INTERVAL;
      
      		reindexUsers(companyId, start, end);
      	}
      }
      
      protected void reindexUsers(long companyId, int start, int end)
      	throws Exception {
      
      	List<User> users = UserLocalServiceUtil.getCompanyUsers(
      		companyId, start, end);
      

      LPS-26223 fixed that by introducing this pattern:

      protected void reindexUsers(long companyId) throws Exception {
      	DynamicQuery dynamicQuery = DynamicQueryFactoryUtil.forClass(
      		User.class, PortalClassLoaderUtil.getClassLoader());
      
      	Projection minUserIdProjection = ProjectionFactoryUtil.min("userId");
      	Projection maxUserIdProjection = ProjectionFactoryUtil.max("userId");
      
      	ProjectionList projectionList = ProjectionFactoryUtil.projectionList();
      
      	projectionList.add(minUserIdProjection);
      	projectionList.add(maxUserIdProjection);
      
      	dynamicQuery.setProjection(projectionList);
      
      	addReindexCriteria(dynamicQuery, companyId);
      
      	List<Object[]> results = UserLocalServiceUtil.dynamicQuery(
      		dynamicQuery);
      
      	Object[] minAndMaxUserIds = results.get(0);
      
      	if ((minAndMaxUserIds[0] == null) || (minAndMaxUserIds[1] == null)) {
      		return;
      	}
      
      	long minUserId = (Long)minAndMaxUserIds[0];
      	long maxUserId = (Long)minAndMaxUserIds[1];
      
      	long startUserId = minUserId;
      	long endUserId = startUserId + DEFAULT_INTERVAL;
      
      	while (startUserId <= maxUserId) {
      		reindexUsers(companyId, startUserId, endUserId);
      
      		startUserId = endUserId;
      		endUserId += DEFAULT_INTERVAL;
      	}
      }
      
      protected void reindexUsers(
      		long companyId, long startUserId, long endUserId)
      	throws Exception {
      
      	DynamicQuery dynamicQuery = DynamicQueryFactoryUtil.forClass(
      		User.class, PortalClassLoaderUtil.getClassLoader());
      
      	Property property = PropertyFactoryUtil.forName("userId");
      
      	dynamicQuery.add(property.ge(startUserId));
      	dynamicQuery.add(property.lt(endUserId));
      
      	addReindexCriteria(dynamicQuery, companyId);
      
      	List<User> users = UserLocalServiceUtil.dynamicQuery(dynamicQuery);
      
      

      But it's still way too verbose, so this ticket introduces a pattern based on a new class called "ActionableDynamicQuery"

      protected void reindexUsers(long companyId)
      	throws PortalException, SystemException {
      
      	final Collection<Document> documents = new ArrayList<Document>();
      
      	ActionableDynamicQuery actionableDynamicQuery =
      		new UserActionableDynamicQuery() {
      
      		@Override
      		protected void performAction(Object object) throws PortalException {
      			User user = (User)object;
      
      			if (!user.isDefaultUser()) {
      				Document document = getDocument(user);
      
      				documents.add(document);
      			}
      		}
      
      	};
      
      	actionableDynamicQuery.setCompanyId(companyId);
      
      	actionableDynamicQuery.performActions();
      
      	SearchEngineUtil.updateDocuments(
      		getSearchEngineId(), companyId, documents);
      }
      

      Much leaner now, another example for Blogs is:

      protected void reindexEntries(long companyId)
      	throws PortalException, SystemException {
      
      	final Collection<Document> documents = new ArrayList<Document>();
      
      	ActionableDynamicQuery actionableDynamicQuery =
      		new BaseActionableDynamicQuery() {
      
      		@Override
      		protected void addCriteria(DynamicQuery dynamicQuery) {
      			Property displayDateProperty = PropertyFactoryUtil.forName(
      				"displayDate");
      
      			dynamicQuery.add(displayDateProperty.lt(new Date()));
      
      			Property statusProperty = PropertyFactoryUtil.forName("status");
      
      			Integer[] statuses = {
      				WorkflowConstants.STATUS_APPROVED,
      				WorkflowConstants.STATUS_IN_TRASH
      			};
      
      			dynamicQuery.add(statusProperty.in(statuses));
      		}
      
      		@Override
      		protected void performAction(Object object) throws PortalException {
      			BlogsEntry entry = (BlogsEntry)object;
      
      			Document document = getDocument(entry);
      
      			documents.add(document);
      		}
      
      	};
      
      	actionableDynamicQuery.setBaseLocalService(
      		BlogsEntryLocalServiceUtil.getService());
      	actionableDynamicQuery.setClass(BlogsEntry.class);
      	actionableDynamicQuery.setClassLoader(
      		PACLClassLoaderUtil.getPortalClassLoader());
      	actionableDynamicQuery.setCompanyId(companyId);
      	actionableDynamicQuery.setPrimaryKeyPropertyName("entryId");
      
      	actionableDynamicQuery.performActions();
      
      	SearchEngineUtil.updateDocuments(
      		getSearchEngineId(), companyId, documents);
      }
      

        Issue Links

          Activity

          Hide
          Randy Zhu added a comment -

          In preparation for Ideation; we are merging New Feature and Improvement tickets into a singular ticket type called “Feature Request”. Additional information to follow soon.

          Show
          Randy Zhu added a comment - In preparation for Ideation; we are merging New Feature and Improvement tickets into a singular ticket type called “Feature Request”. Additional information to follow soon.
          Hide
          Laszlo Csontos (Inactive) added a comment -

          Committed on:

          Portal EE 6.1.x GIT ID: 27c140c189d3d0d57c0ae1d590af2164febbd9bf.
          Portal CE 6.2.x GIT ID: 5191d50ac2dafdf077afa3a023965fdd19bc5b6b.

          Show
          Laszlo Csontos (Inactive) added a comment - Committed on: Portal EE 6.1.x GIT ID: 27c140c189d3d0d57c0ae1d590af2164febbd9bf. Portal CE 6.2.x GIT ID: 5191d50ac2dafdf077afa3a023965fdd19bc5b6b.

            People

            • Votes:
              0 Vote for this issue
              Watchers:
              2 Start watching this issue

              Dates

              • Created:
                Updated:
                Resolved:

                Development

                  Subcomponents

                    Structure Helper Panel