WebSVN – SmartDukaan – Diff – /trunk/GoogleAppDatalog/src/main/java/in/shop2020/web/DailyProductWithSourceAggregatorServlet.java

     private static final String EVENTTYPE_FIELD = "eventType";
     private static final String DATE_FIELD = "date";
     private static final String FIRST_SOURCE_FIELD = "firstSource";
     private static final String SESSION_SOURCE_FIELD = "source";
     private static final String PRODUCT_VIEW_EVENT = "PRODUCT_VIEW";
     private static final String ADD_TO_CART_EVENT = "ADD_TO_CART";
     private static final String DELETE_FROM_CART_EVENT = "DELETE_FROM_CART";
     private static final String RESEARCH_ADD_EVENT = "RESEARCH_ADD";
     private static final String RESEARCH_DELETE_EVENT = "RESEARCH_DELETE";
     private Map<String, Map<String, String>> sessionIdSourceMap;
     private Date fromDate;
     private Date toDate;
+    private Date queryFromDate;
+    private Date queryToDate;
     private DatastoreService datastore;
     public void doPost(HttpServletRequest req, HttpServletResponse resp) {
         logger.warning("Running DailyProductWithSourceAggregator");
         sessionIdSourceMap = new HashMap<String, Map<String,String>>();
             itemsMap.put(item.getId(), item);
             itemsCatalogIdMap.put(item.getCatalogId(), item);
             itemsNameMap.put(item.getName().trim(), item);
+        }
-        logger.warning("DailyProductWithSourceAggregator : Updating View Count Map");
-        UpdateViewCount(PRODUCT_VIEW_EVENT, VIEW_COUNT_KEY, UNIQUE_VIEW_COUNT_KEY);
-        logger.warning("DailyProductWithSourceAggregator : itemsResultMap size : " + itemsResultMap.size());
-        logger.warning("DailyProductWithSourceAggregator : sessionIdSourceMap size : " + sessionIdSourceMap.size());
-        logger.warning("DailyProductWithSourceAggregator : Updating Add To Cart Count Map");
-        UpdateItemIdEventCount(ADD_TO_CART_EVENT, ADD_TO_CART_COUNT_KEY, UNIQUE_ADD_TO_CART_COUNT_KEY, false);
-        logger.warning("DailyProductWithSourceAggregator : Updating Delete From Cart Count Map");
-        UpdateItemIdEventCount(DELETE_FROM_CART_EVENT, DELETE_FROM_CART_COUNT_KEY, UNIQUE_DELETE_FROM_CART_COUNT_KEY, false);
-        logger.warning("DailyProductWithSourceAggregator : Updating Add To Research Count Map");
-        UpdateItemIdEventCount(RESEARCH_ADD_EVENT, ADD_TO_RESEARCH_COUNT_KEY, UNIQUE_ADD_TO_RESEARCH_COUNT_KEY, true);
-        logger.warning("DailyProductWithSourceAggregator : Updating Delete From Research Count Map");
-        UpdateItemIdEventCount(RESEARCH_DELETE_EVENT, DELETE_FROM_RESEARCH_COUNT_KEY, UNIQUE_DELETE_FROM_RESEARCH_COUNT_KEY, true);
-        logger.warning("DailyProductWithSourceAggregator : Updating ShippingAccessCount Count Map");
-        UpdateItemIdEventCount(SHIPPING_ACCESS_EVENT, SHIPPING_ACCESS_COUNT_KEY, UNIQUE_SHIPPING_ACCESS_COUNT_KEY, false);
-        logger.warning("DailyProductWithSourceAggregator : Updating Payment Success Count Map");
-        UpdateItemIdEventCount(PAYMENT_SUCCESS_EVENT, PAYMENT_SUCCESS_COUNT_KEY, UNIQUE_PAYMENT_SUCCESS_COUNT_KEY, false);
-        logger.warning("DailyProductWithSourceAggregator : Updating Order Creation Count Map");
-        UpdateItemIdEventCount(ORDER_CREATION_EVENT, ORDER_CREATION_COUNT_KEY, UNIQUE_ORDER_CREATION_COUNT_KEY, false);
-        logger.warning("DailyProductWithSourceAggregator : Updating Payment Failure Count Map");
-        UpdateItemIdEventCount(PAYMENT_FAILURE_EVENT, PAYMENT_FAILURE_COUNT_KEY, UNIQUE_PAYMENT_FAILURE_COUNT_KEY, false);
-        logger.warning("DailyProductWithSourceAggregator : Updating Proceed To Pay Count Map");
-        UpdateItemIdEventCount(PROCEED_TO_PAY_EVENT, PROCEED_TO_PAY_COUNT_KEY, UNIQUE_PROCEED_TO_PAY_COUNT_KEY, false);
-        List<ItemActivityWithSource> itemActivities = new ArrayList<ItemActivityWithSource>();
-        ItemActivityWithSourceRepository itemActivityRepository = new ItemActivityWithSourceRepository();
-        for (Entry<String, Map<String, Long>> entry : itemsResultMap.entrySet()) {
+        for (int i = 0; i < 4; i++) {
-            String key = (String)entry.getKey();
+            cal.setTime(fromDate);
-            Map<String, Long> itemMap = (Map<String, Long>)entry.getValue();
+            cal.add(Calendar.HOUR_OF_DAY, 6*i);
-            String [] keyItems = key.split("_");
+            queryFromDate = cal.getTime();
-            Boolean isPaidFirstSource = keyItems[0].equals("0")? false : true;
-            String firstSource = keyItems[1];
+            cal.add(Calendar.HOUR_OF_DAY, 6);
-            Boolean isPaidSessionSource = keyItems[2].equals("0")? false : true;
-            String sessionSource = keyItems[3];
+            queryToDate = cal.getTime();
-            Long catalogItemId = Long.parseLong(keyItems[4]);
-            ItemActivityWithSource itemActivity = new ItemActivityWithSource();
-            itemActivity.setIsPaidFirstSource(isPaidFirstSource);
+            logger.warning("From Query Date : " + queryFromDate);
-            itemActivity.setFirstSource(firstSource);
-            itemActivity.setIsPaidSessionSource(isPaidSessionSource);
-            itemActivity.setSessionSource(sessionSource);
-            itemActivity.setCatalogId(catalogItemId);
-            itemActivity.setDate(fromDate);
+            logger.warning("To Query Date : " + queryToDate);
+            logger.warning("DailyProductWithSourceAggregator : Updating View Count Map");
+            UpdateViewCount(PRODUCT_VIEW_EVENT, VIEW_COUNT_KEY,
+                    UNIQUE_VIEW_COUNT_KEY);
+            logger.warning("DailyProductWithSourceAggregator : itemsResultMap size : "
+                    + itemsResultMap.size());
+            logger.warning("DailyProductWithSourceAggregator : sessionIdSourceMap size : "
+                    + sessionIdSourceMap.size());
+            logger.warning("DailyProductWithSourceAggregator : Updating Add To Cart Count Map");
+            UpdateItemIdEventCount(ADD_TO_CART_EVENT, ADD_TO_CART_COUNT_KEY,
+                    UNIQUE_ADD_TO_CART_COUNT_KEY, false);
+            logger.warning("DailyProductWithSourceAggregator : Updating Delete From Cart Count Map");
+            UpdateItemIdEventCount(DELETE_FROM_CART_EVENT,
+                    DELETE_FROM_CART_COUNT_KEY,
+                    UNIQUE_DELETE_FROM_CART_COUNT_KEY, false);
+            logger.warning("DailyProductWithSourceAggregator : Updating Add To Research Count Map");
+            UpdateItemIdEventCount(RESEARCH_ADD_EVENT,
+                    ADD_TO_RESEARCH_COUNT_KEY,
+                    UNIQUE_ADD_TO_RESEARCH_COUNT_KEY, true);
+            logger.warning("DailyProductWithSourceAggregator : Updating Delete From Research Count Map");
+            UpdateItemIdEventCount(RESEARCH_DELETE_EVENT,
+                    DELETE_FROM_RESEARCH_COUNT_KEY,
+                    UNIQUE_DELETE_FROM_RESEARCH_COUNT_KEY, true);
+            logger.warning("DailyProductWithSourceAggregator : Updating ShippingAccessCount Count Map");
+            UpdateItemIdEventCount(SHIPPING_ACCESS_EVENT,
+                    SHIPPING_ACCESS_COUNT_KEY,
+                    UNIQUE_SHIPPING_ACCESS_COUNT_KEY, false);
+            logger.warning("DailyProductWithSourceAggregator : Updating Payment Success Count Map");
+            UpdateItemIdEventCount(PAYMENT_SUCCESS_EVENT,
+                    PAYMENT_SUCCESS_COUNT_KEY,
+                    UNIQUE_PAYMENT_SUCCESS_COUNT_KEY, false);
+            logger.warning("DailyProductWithSourceAggregator : Updating Order Creation Count Map");
+            UpdateItemIdEventCount(ORDER_CREATION_EVENT,
+                    ORDER_CREATION_COUNT_KEY, UNIQUE_ORDER_CREATION_COUNT_KEY,
+                    false);
+            logger.warning("DailyProductWithSourceAggregator : Updating Payment Failure Count Map");
+            UpdateItemIdEventCount(PAYMENT_FAILURE_EVENT,
+                    PAYMENT_FAILURE_COUNT_KEY,
+                    UNIQUE_PAYMENT_FAILURE_COUNT_KEY, false);
+            logger.warning("DailyProductWithSourceAggregator : Updating Proceed To Pay Count Map");
+            UpdateItemIdEventCount(PROCEED_TO_PAY_EVENT,
+                    PROCEED_TO_PAY_COUNT_KEY, UNIQUE_PROCEED_TO_PAY_COUNT_KEY,
+                    false);
+            List<ItemActivityWithSource> itemActivities = new ArrayList<ItemActivityWithSource>();
+            ItemActivityWithSourceRepository itemActivityRepository = new ItemActivityWithSourceRepository();
+            for (Entry<String, Map<String, Long>> entry : itemsResultMap
+                    .entrySet()) {
+                String key = (String) entry.getKey();
+                Map<String, Long> itemMap = (Map<String, Long>) entry
+                        .getValue();
+                String[] keyItems = key.split("_");
+                Boolean isPaidFirstSource = keyItems[0].equals("0") ? false
+                        : true;
+                String firstSource = keyItems[1];
+                Boolean isPaidSessionSource = keyItems[2].equals("0") ? false
+                        : true;
+                String sessionSource = keyItems[3];
+                Long catalogItemId = Long.parseLong(keyItems[4]);
+                ItemActivityWithSource itemActivity = new ItemActivityWithSource();
+                itemActivity.setIsPaidFirstSource(isPaidFirstSource);
+                itemActivity.setFirstSource(firstSource);
+                itemActivity.setIsPaidSessionSource(isPaidSessionSource);
+                itemActivity.setSessionSource(sessionSource);
+                itemActivity.setCatalogId(catalogItemId);
+                itemActivity.setDate(fromDate);
-            itemActivity.setView(itemMap.get(VIEW_COUNT_KEY));
+                itemActivity.setView(itemMap.get(VIEW_COUNT_KEY));
-            itemActivity.setAddToCart(itemMap.get(ADD_TO_CART_COUNT_KEY));
+                itemActivity.setAddToCart(itemMap.get(ADD_TO_CART_COUNT_KEY));
+                itemActivity.setAddToResearch(itemMap
-            itemActivity.setAddToResearch(itemMap.get(ADD_TO_RESEARCH_COUNT_KEY));
+                        .get(ADD_TO_RESEARCH_COUNT_KEY));
+                itemActivity.setDeleteFromCart(itemMap
-            itemActivity.setDeleteFromCart(itemMap.get(DELETE_FROM_CART_COUNT_KEY));
+                        .get(DELETE_FROM_CART_COUNT_KEY));
+                itemActivity.setDeleteFromResearch(itemMap
-            itemActivity.setDeleteFromResearch(itemMap.get(DELETE_FROM_RESEARCH_COUNT_KEY));
+                        .get(DELETE_FROM_RESEARCH_COUNT_KEY));
-            itemActivity.setShippingAccess(itemMap.get(SHIPPING_ACCESS_COUNT_KEY));
+                itemActivity.setShippingAccess(itemMap
+                        .get(SHIPPING_ACCESS_COUNT_KEY));
-            itemActivity.setPaymentSuccess(itemMap.get(PAYMENT_SUCCESS_COUNT_KEY));
+                itemActivity.setPaymentSuccess(itemMap
+                        .get(PAYMENT_SUCCESS_COUNT_KEY));
-            itemActivity.setOrderCreation(itemMap.get(ORDER_CREATION_COUNT_KEY));
+                itemActivity.setOrderCreation(itemMap
+                        .get(ORDER_CREATION_COUNT_KEY));
-            itemActivity.setPaymentFailure(itemMap.get(PAYMENT_FAILURE_COUNT_KEY));
+                itemActivity.setPaymentFailure(itemMap
+                        .get(PAYMENT_FAILURE_COUNT_KEY));
-            itemActivity.setProceedToPay(itemMap.get(PROCEED_TO_PAY_COUNT_KEY));
+                itemActivity.setProceedToPay(itemMap
+                        .get(PROCEED_TO_PAY_COUNT_KEY));
-            itemActivity.setUniqueView(itemMap.get(UNIQUE_VIEW_COUNT_KEY));
+                itemActivity.setUniqueView(itemMap.get(UNIQUE_VIEW_COUNT_KEY));
+                itemActivity.setUniqueAddToCart(itemMap
-            itemActivity.setUniqueAddToCart(itemMap.get(UNIQUE_ADD_TO_CART_COUNT_KEY));
+                        .get(UNIQUE_ADD_TO_CART_COUNT_KEY));
+                itemActivity.setUniqueAddToResearch(itemMap
-            itemActivity.setUniqueAddToResearch(itemMap.get(UNIQUE_ADD_TO_RESEARCH_COUNT_KEY));
+                        .get(UNIQUE_ADD_TO_RESEARCH_COUNT_KEY));
+                itemActivity.setUniqueDeleteFromCart(itemMap
-            itemActivity.setUniqueDeleteFromCart(itemMap.get(UNIQUE_DELETE_FROM_CART_COUNT_KEY));
+                        .get(UNIQUE_DELETE_FROM_CART_COUNT_KEY));
-            itemActivity.setUniqueDeleteFromResearch(itemMap.get(UNIQUE_ADD_TO_RESEARCH_COUNT_KEY));
+                itemActivity.setUniqueDeleteFromResearch(itemMap
-            itemActivity.setUniqueShippingAccess(itemMap.get(UNIQUE_SHIPPING_ACCESS_COUNT_KEY));
+                        .get(UNIQUE_ADD_TO_RESEARCH_COUNT_KEY));
-            itemActivity.setUniquePaymentSuccess(itemMap.get(UNIQUE_PAYMENT_SUCCESS_COUNT_KEY));
+                itemActivity.setUniqueShippingAccess(itemMap
-            itemActivity.setUniqueOrderCreation(itemMap.get(UNIQUE_ORDER_CREATION_COUNT_KEY));
+                        .get(UNIQUE_SHIPPING_ACCESS_COUNT_KEY));
-            itemActivity.setUniquePaymentFailure(itemMap.get(UNIQUE_PAYMENT_FAILURE_COUNT_KEY));
+                itemActivity.setUniquePaymentSuccess(itemMap
-            itemActivity.setUniqueProceedToPay(itemMap.get(UNIQUE_PROCEED_TO_PAY_COUNT_KEY));
+                        .get(UNIQUE_PAYMENT_SUCCESS_COUNT_KEY));
-            itemActivities.add(itemActivity);
+                itemActivity.setUniqueOrderCreation(itemMap
+                        .get(UNIQUE_ORDER_CREATION_COUNT_KEY));
-            if (itemActivities.size()>500) {
+                itemActivity.setUniquePaymentFailure(itemMap
-                logger.warning("DailyProductWithSourceAggregator : persisting itemactivities : " + itemActivities.size());
+                        .get(UNIQUE_PAYMENT_FAILURE_COUNT_KEY));
-                itemActivityRepository.createAll(itemActivities);
+                itemActivity.setUniqueProceedToPay(itemMap
+                        .get(UNIQUE_PROCEED_TO_PAY_COUNT_KEY));
-                itemActivities.clear();
+                itemActivities.add(itemActivity);
+            }
+            logger.warning("DailyProductWithSourceAggregator : persisting itemactivities : "
+                    + itemActivities.size());
+            logger.warning("DailyProductWithSourceAggregator : itemactivities : "
+                    + itemActivities);
+            itemActivityRepository.createAll(itemActivities);
+        }
-        logger.warning("DailyProductWithSourceAggregator : persisting itemactivities : " + itemActivities.size());
-        logger.warning("DailyProductWithSourceAggregator : itemactivities : " + itemActivities);
-        itemActivityRepository.createAll(itemActivities);
+    }
     public void doGet(HttpServletRequest req, HttpServletResponse resp) {
         doPost(req, resp);
+    }
     private void UpdateViewCount(String event, String viewCountKey, String uniqueViewCountKey) {
         Query query = new Query(KIND);
         query.addFilter(EVENTTYPE_FIELD, Query.FilterOperator.EQUAL, event);
-        query.addFilter(DATE_FIELD, Query.FilterOperator.GREATER_THAN_OR_EQUAL, fromDate);
+        query.addFilter(DATE_FIELD, Query.FilterOperator.GREATER_THAN_OR_EQUAL, queryFromDate);
-        query.addFilter(DATE_FIELD, Query.FilterOperator.LESS_THAN, toDate);
+        query.addFilter(DATE_FIELD, Query.FilterOperator.LESS_THAN, queryToDate);
         PreparedQuery pq = datastore.prepare(query);
         Set<String> uniqueSet = new HashSet<String>();
         List<Entity> entityResults = new ArrayList<Entity>();
         Set<String> sessionIds = new HashSet<String>();
         logger.warning("DailyProductWithSourceAggregator : Getting View Events in list");
-        for (Entity e : pq.asList(FetchOptions.Builder.withChunkSize(500))) {
+        for (Entity e : pq.asIterable(FetchOptions.Builder.withChunkSize(500))) {
             entityResults.add(e);
+        }
+        for (Entity e : entityResults) {
             String sessionId = (String)e.getProperty(SESSION_ID_FIELD);
             if (sessionId != null && !sessionId.isEmpty()) {
                 sessionIds.add(sessionId);
+            }
+        }
     private void populateSessionIdSourceMap(Set<String> sessionIds) {
         Query sessionQuery = new Query(KIND);
         sessionQuery.addFilter(EVENTTYPE_FIELD, Query.FilterOperator.EQUAL,
                 NEW_SESSION_EVENT);
-        sessionQuery.addFilter(DATE_FIELD, Query.FilterOperator.GREATER_THAN_OR_EQUAL, fromDate);
+        sessionQuery.addFilter(DATE_FIELD, Query.FilterOperator.GREATER_THAN_OR_EQUAL, queryFromDate);
-        sessionQuery.addFilter(DATE_FIELD, Query.FilterOperator.LESS_THAN, toDate);
+        sessionQuery.addFilter(DATE_FIELD, Query.FilterOperator.LESS_THAN, queryToDate);
         PreparedQuery sessionPq = datastore.prepare(sessionQuery);
+        List<Entity> entityResults = new ArrayList<Entity>();
+        for (Entity e : sessionPq.asIterable(FetchOptions.Builder.withChunkSize(500))) {
+            entityResults.add(e);
+        }
-        for (Entity sessionResult : sessionPq.asIterable()) {
+        for (Entity sessionResult : entityResults) {
             String sessionId = (String)sessionResult.getProperty(SESSION_ID_FIELD);
             if (sessionIds.contains(sessionId)) {
                 String firstSource = "";
                 String sessionSource = "";
                 if (sessionResult != null) {
+    }
     private void UpdateItemIdEventCount(String event, String countKey, String uniqueCountKey, boolean isCatalogIdEvent) {
         Query query = new Query(KIND);
         query.addFilter(EVENTTYPE_FIELD, Query.FilterOperator.EQUAL, event);
-        query.addFilter(DATE_FIELD, Query.FilterOperator.GREATER_THAN_OR_EQUAL, fromDate);
+        query.addFilter(DATE_FIELD, Query.FilterOperator.GREATER_THAN_OR_EQUAL, queryFromDate);
-        query.addFilter(DATE_FIELD, Query.FilterOperator.LESS_THAN, toDate);
+        query.addFilter(DATE_FIELD, Query.FilterOperator.LESS_THAN, queryToDate);
         PreparedQuery pq = datastore.prepare(query);
         Set<String> uniqueSet = new HashSet<String>();
         List<Entity> results = new ArrayList<Entity>();
         for (Entity e : pq.asList(FetchOptions.Builder.withChunkSize(500))) {
             results.add(e);

Subversion Repositories SmartDukaan

(root)/trunk/GoogleAppDatalog/src/main/java/in/shop2020/web/DailyProductWithSourceAggregatorServlet.java – Rev 3523 → 3531