WebSVN – SmartDukaan – Diff – /trunk/GoogleAppDatalog/src/main/java/in/shop2020/web/DailyProductWithSourceAggregatorServlet.java

 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 import java.util.TimeZone;
+import java.util.logging.Logger;
 import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import com.google.appengine.api.datastore.DatastoreService;
 import com.google.appengine.api.datastore.DatastoreServiceFactory;
 import com.google.appengine.api.datastore.Entity;
+import com.google.appengine.api.datastore.FetchOptions;
 import com.google.appengine.api.datastore.PreparedQuery;
 import com.google.appengine.api.datastore.Query;
 public class DailyProductWithSourceAggregatorServlet extends HttpServlet {
     private static final long serialVersionUID = -8236918415987438049L;
+    private static Logger logger = Logger.getLogger(DailyProductWithSourceAggregatorServlet.class.getName());
     private static final String KIND = "DataLog";
-    private static final String SESSION_ID_FILED = "sessionId";
+    private static final String SESSION_ID_FIELD = "sessionId";
-    private static final String ITEMID_FILED = "itemId";
+    private static final String ITEMID_FIELD = "itemId";
-    private static final String ITEMIDS_FILED = "itemIds";
+    private static final String ITEMIDS_FIELD = "itemIds";
-    private static final String PRODUCTNAME_FILED = "productName";
+    private static final String PRODUCTNAME_FIELD = "productName";
-    private static final String EVENTTYPE_FILED = "eventType";
+    private static final String EVENTTYPE_FIELD = "eventType";
-    private static final String DATE_FILED = "date";
+    private static final String DATE_FIELD = "date";
-    private static final String FIRST_SOURCE_FILED = "firstSource";
+    private static final String FIRST_SOURCE_FIELD = "firstSource";
-    private static final String SESSION_SOURCE_FILED = "source";
+    private static final String SESSION_SOURCE_FIELD = "source";
     private static final String PRODUCT_VIEW_EVENT = "PRODUCT_VIEW";
     private static final String ADD_TO_CART_EVENT = "ADD_TO_CART";
     private static final String DELETE_FROM_CART_EVENT = "DELETE_FROM_CART";
     private static final String UNIQUE_PROCEED_TO_PAY_COUNT_KEY = "UniqueProceedToPayCount";
     private static final String FIRST_SOURCE_KEY = "FirstSource";
     private static final String SESSION_SOURCE_KEY = "SessionSource";
-    private Map<Long, Item> itemsMap = new HashMap<Long, Item>();
+    private Map<Long, Item> itemsMap;
-    private Map<Long, Item> itemsCatalogIdMap = new HashMap<Long, Item>();
+    private Map<Long, Item> itemsCatalogIdMap;
-    private Map<String, Item> itemsNameMap = new HashMap<String, Item>();
+    private Map<String, Item> itemsNameMap;
-    private Map<String, Map<String, Long>> itemsResultMap = new HashMap<String, Map<String, Long>>();
+    private Map<String, Map<String, Long>> itemsResultMap;
-    private Map<String, Map<String, String>> sessionIdSourceMap = new HashMap<String, Map<String,String>>();
+    private Map<String, Map<String, String>> sessionIdSourceMap;
     private Date fromDate;
     private Date toDate;
     private DatastoreService datastore;
     public void doPost(HttpServletRequest req, HttpServletResponse resp) {
+        logger.warning("Running DailyProductWithSourceAggregator");
+        sessionIdSourceMap = new HashMap<String, Map<String,String>>();
+        itemsMap = new HashMap<Long, Item>();
+        itemsCatalogIdMap = new HashMap<Long, Item>();
+        itemsNameMap = new HashMap<String, Item>();
+        itemsResultMap = new HashMap<String, Map<String, Long>>();
         datastore = DatastoreServiceFactory.getDatastoreService();
         SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
         Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("IST"));
         String dateStr = req.getParameter("date");
         if (dateStr != null && !dateStr.isEmpty()) {
             itemsMap.put(item.getId(), item);
             itemsCatalogIdMap.put(item.getCatalogId(), item);
             itemsNameMap.put(item.getName().trim(), item);
+        }
+        logger.warning("DailyProductWithSourceAggregator : Updating View Count Map");
         UpdateViewCount(PRODUCT_VIEW_EVENT, VIEW_COUNT_KEY, UNIQUE_VIEW_COUNT_KEY);
+        logger.warning("DailyProductWithSourceAggregator : itemsResultMap size : " + itemsResultMap.size());
+        logger.warning("DailyProductWithSourceAggregator : sessionIdSourceMap size : " + sessionIdSourceMap.size());
+        logger.warning("DailyProductWithSourceAggregator : Updating Add To Cart Count Map");
         UpdateItemIdEventCount(ADD_TO_CART_EVENT, ADD_TO_CART_COUNT_KEY, UNIQUE_ADD_TO_CART_COUNT_KEY, false);
+        logger.warning("DailyProductWithSourceAggregator : Updating Delete From Cart Count Map");
         UpdateItemIdEventCount(DELETE_FROM_CART_EVENT, DELETE_FROM_CART_COUNT_KEY, UNIQUE_DELETE_FROM_CART_COUNT_KEY, false);
+        logger.warning("DailyProductWithSourceAggregator : Updating Add To Research Count Map");
         UpdateItemIdEventCount(RESEARCH_ADD_EVENT, ADD_TO_RESEARCH_COUNT_KEY, UNIQUE_ADD_TO_RESEARCH_COUNT_KEY, true);
+        logger.warning("DailyProductWithSourceAggregator : Updating Delete From Research Count Map");
         UpdateItemIdEventCount(RESEARCH_DELETE_EVENT, DELETE_FROM_RESEARCH_COUNT_KEY, UNIQUE_DELETE_FROM_RESEARCH_COUNT_KEY, true);
+        logger.warning("DailyProductWithSourceAggregator : Updating ShippingAccessCount Count Map");
         UpdateItemIdEventCount(SHIPPING_ACCESS_EVENT, SHIPPING_ACCESS_COUNT_KEY, UNIQUE_SHIPPING_ACCESS_COUNT_KEY, false);
+        logger.warning("DailyProductWithSourceAggregator : Updating Payment Success Count Map");
         UpdateItemIdEventCount(PAYMENT_SUCCESS_EVENT, PAYMENT_SUCCESS_COUNT_KEY, UNIQUE_PAYMENT_SUCCESS_COUNT_KEY, false);
+        logger.warning("DailyProductWithSourceAggregator : Updating Order Creation Count Map");
         UpdateItemIdEventCount(ORDER_CREATION_EVENT, ORDER_CREATION_COUNT_KEY, UNIQUE_ORDER_CREATION_COUNT_KEY, false);
+        logger.warning("DailyProductWithSourceAggregator : Updating Payment Failure Count Map");
         UpdateItemIdEventCount(PAYMENT_FAILURE_EVENT, PAYMENT_FAILURE_COUNT_KEY, UNIQUE_PAYMENT_FAILURE_COUNT_KEY, false);
+        logger.warning("DailyProductWithSourceAggregator : Updating Proceed To Pay Count Map");
         UpdateItemIdEventCount(PROCEED_TO_PAY_EVENT, PROCEED_TO_PAY_COUNT_KEY, UNIQUE_PROCEED_TO_PAY_COUNT_KEY, false);
         List<ItemActivityWithSource> itemActivities = new ArrayList<ItemActivityWithSource>();
         ItemActivityWithSourceRepository itemActivityRepository = new ItemActivityWithSourceRepository();
         for (Entry<String, Map<String, Long>> entry : itemsResultMap.entrySet()) {
             itemActivity.setUniqueOrderCreation(itemMap.get(UNIQUE_ORDER_CREATION_COUNT_KEY));
             itemActivity.setUniquePaymentFailure(itemMap.get(UNIQUE_PAYMENT_FAILURE_COUNT_KEY));
             itemActivity.setUniqueProceedToPay(itemMap.get(UNIQUE_PROCEED_TO_PAY_COUNT_KEY));
             itemActivities.add(itemActivity);
+            if (itemActivities.size()>500) {
+                logger.warning("DailyProductWithSourceAggregator : persisting itemactivities : " + itemActivities.size());
+                itemActivityRepository.createAll(itemActivities);
+                itemActivities.clear();
+            }
+        }
+        logger.warning("DailyProductWithSourceAggregator : persisting itemactivities : " + itemActivities.size());
+        logger.warning("DailyProductWithSourceAggregator : itemactivities : " + itemActivities);
         itemActivityRepository.createAll(itemActivities);
+    }
     public void doGet(HttpServletRequest req, HttpServletResponse resp) {
         doPost(req, resp);
+    }
     private void UpdateViewCount(String event, String viewCountKey, String uniqueViewCountKey) {
         Query query = new Query(KIND);
-        query.addFilter(EVENTTYPE_FILED, Query.FilterOperator.EQUAL, event);
+        query.addFilter(EVENTTYPE_FIELD, Query.FilterOperator.EQUAL, event);
-        query.addFilter(DATE_FILED, Query.FilterOperator.GREATER_THAN_OR_EQUAL, fromDate);
+        query.addFilter(DATE_FIELD, Query.FilterOperator.GREATER_THAN_OR_EQUAL, fromDate);
-        query.addFilter(DATE_FILED, Query.FilterOperator.LESS_THAN, toDate);
+        query.addFilter(DATE_FIELD, Query.FilterOperator.LESS_THAN, toDate);
         PreparedQuery pq = datastore.prepare(query);
         Set<String> uniqueSet = new HashSet<String>();
+        List<Entity> entityResults = new ArrayList<Entity>();
+        logger.warning("DailyProductWithSourceAggregator : Getting View Events in list");
+        for (Entity e : pq.asList(FetchOptions.Builder.withChunkSize(500))) {
+            entityResults.add(e);
+        }
+        logger.warning("DailyProductWithSourceAggregator : Done Getting View Events in list");
-        for (Entity result : pq.asIterable()) {
+        for (Entity result : entityResults) {
-            String sessionId = (String)result.getProperty(SESSION_ID_FILED);
+            String sessionId = (String)result.getProperty(SESSION_ID_FIELD);
             String firstSource = getSource(sessionId, FIRST_SOURCE_KEY);
             String sessionSource = getSource(sessionId, SESSION_SOURCE_KEY);
             String paidFirstSource = "0";
             String paidSessionSource = "0";
             if (firstSource.startsWith("PAID :")) {
                 paidFirstSource = "1";
+            }
             if (sessionSource.startsWith("PAID :")) {
                 paidSessionSource = "1";
+            }
-            firstSource = firstSource.replaceAll(".*http[s]?://", "").replaceAll("/.*", "");
+            firstSource = firstSource.replaceAll(".*http[s]?://", "").replaceAll("/.*", "").replaceAll("www.", "");
-            sessionSource = sessionSource.replaceAll(".*http[s]?://", "").replaceAll("/.*", "");
+            sessionSource = sessionSource.replaceAll(".*http[s]?://", "").replaceAll("/.*", "").replaceAll("www.", "");
-            Long catalogItemId = (Long)result.getProperty(ITEMID_FILED);
+            Long catalogItemId = (Long)result.getProperty(ITEMID_FIELD);
             if(catalogItemId == null) {
-                String name = (String)result.getProperty(PRODUCTNAME_FILED);
+                String name = (String)result.getProperty(PRODUCTNAME_FIELD);
                 if (itemsNameMap.containsKey(name.trim())) {
                     catalogItemId = itemsNameMap.get(name.trim()).getCatalogId();
+                }
+            }
             String key = paidFirstSource + "_" + firstSource + "_"
+            }
+        }
+    }
     private String getSource(String sessionId, String sourceKey) {
-        if (sessionIdSourceMap.isEmpty()) {
+        if (sessionIdSourceMap.containsKey(sessionId)) {
+            logger.warning("DailyProductWithSourceAggregator : sessionIdSourceMap hit : "
+                    + sessionId);
+            return sessionIdSourceMap.get(sessionId).get(sourceKey);
+        }
+        if (sessionId != null && !sessionId.isEmpty()) {
             Query sessionQuery = new Query(KIND);
-            sessionQuery.addFilter(EVENTTYPE_FILED, Query.FilterOperator.EQUAL,
+            sessionQuery.addFilter(EVENTTYPE_FIELD, Query.FilterOperator.EQUAL,
                     NEW_SESSION_EVENT);
-            sessionQuery.addFilter(DATE_FILED, Query.FilterOperator.GREATER_THAN_OR_EQUAL, fromDate);
+            sessionQuery.addFilter(SESSION_ID_FIELD,
-            sessionQuery.addFilter(DATE_FILED, Query.FilterOperator.LESS_THAN, toDate);
+                    Query.FilterOperator.EQUAL, sessionId);
             PreparedQuery sessionPq = datastore.prepare(sessionQuery);
             for (Entity sessionResult : sessionPq.asIterable()) {
                 String firstSource = "";
                 String sessionSource = "";
                 if (sessionResult != null) {
-                    if (sessionResult.hasProperty(FIRST_SOURCE_FILED)) {
+                    if (sessionResult.hasProperty(FIRST_SOURCE_FIELD)) {
-                        firstSource = sessionResult.getProperty(FIRST_SOURCE_FILED).toString();
+                        firstSource = sessionResult.getProperty(
+                                FIRST_SOURCE_FIELD).toString();
+                    }
-                    if (sessionResult.hasProperty(SESSION_SOURCE_FILED)) {
+                    if (sessionResult.hasProperty(SESSION_SOURCE_FIELD)) {
-                        sessionSource = sessionResult.getProperty(SESSION_SOURCE_FILED).toString();
+                        sessionSource = sessionResult.getProperty(
+                                SESSION_SOURCE_FIELD).toString();
+                    }
+                }
                 Map<String, String> sessionSourcesMap = new HashMap<String, String>();
                 sessionSourcesMap.put(FIRST_SOURCE_KEY, firstSource);
                 sessionSourcesMap.put(SESSION_SOURCE_KEY, sessionSource);
+            }
+        }
         if (sessionIdSourceMap.containsKey(sessionId)) {
             return sessionIdSourceMap.get(sessionId).get(sourceKey);
+        }
-        Query sessionQuery = new Query(KIND);
-        sessionQuery.addFilter(EVENTTYPE_FILED, Query.FilterOperator.EQUAL,
-                NEW_SESSION_EVENT);
-        sessionQuery.addFilter(SESSION_ID_FILED, Query.FilterOperator.EQUAL, sessionId);
-        PreparedQuery sessionPq = datastore.prepare(sessionQuery);
-        for (Entity sessionResult : sessionPq.asIterable()) {
-            String firstSource = "";
-            String sessionSource = "";
-            if (sessionResult != null) {
-                if (sessionResult.hasProperty(FIRST_SOURCE_FILED)) {
-                    firstSource = sessionResult.getProperty(FIRST_SOURCE_FILED).toString();
-                }
-                if (sessionResult.hasProperty(SESSION_SOURCE_FILED)) {
-                    sessionSource = sessionResult.getProperty(SESSION_SOURCE_FILED).toString();
-                }
-            }
-            Map<String, String> sessionSourcesMap = new HashMap<String, String>();
-            sessionSourcesMap.put(FIRST_SOURCE_KEY, firstSource);
-            sessionSourcesMap.put(SESSION_SOURCE_KEY, sessionSource);
-            sessionIdSourceMap.put(sessionId, sessionSourcesMap);
-        }
-        if (sessionIdSourceMap.containsKey(sessionId)) {
-            return sessionIdSourceMap.get(sessionId).get(sourceKey);
-        }
         return "";
+    }
     private void UpdateItemIdEventCount(String event, String countKey, String uniqueCountKey, boolean isCatalogIdEvent) {
         Query query = new Query(KIND);
-        query.addFilter(EVENTTYPE_FILED, Query.FilterOperator.EQUAL, event);
+        query.addFilter(EVENTTYPE_FIELD, Query.FilterOperator.EQUAL, event);
-        query.addFilter(DATE_FILED, Query.FilterOperator.GREATER_THAN_OR_EQUAL, fromDate);
+        query.addFilter(DATE_FIELD, Query.FilterOperator.GREATER_THAN_OR_EQUAL, fromDate);
-        query.addFilter(DATE_FILED, Query.FilterOperator.LESS_THAN, toDate);
+        query.addFilter(DATE_FIELD, Query.FilterOperator.LESS_THAN, toDate);
         PreparedQuery pq = datastore.prepare(query);
         Set<String> uniqueSet = new HashSet<String>();
+        List<Entity> results = new ArrayList<Entity>();
+        for (Entity e : pq.asList(FetchOptions.Builder.withChunkSize(500))) {
+            results.add(e);
+        }
-        for (Entity result : pq.asIterable()) {
+        for (Entity result : results) {
-            String sessionId = (String)result.getProperty(SESSION_ID_FILED);
+            String sessionId = (String)result.getProperty(SESSION_ID_FIELD);
             String firstSource = getSource(sessionId, FIRST_SOURCE_KEY);
             String sessionSource = getSource(sessionId, SESSION_SOURCE_KEY);
             String paidFirstSource = "0";
             String paidSessionSource = "0";
             if (firstSource.startsWith("PAID :")) {
                 paidFirstSource = "1";
+            }
             if (sessionSource.startsWith("PAID :")) {
                 paidSessionSource = "1";
+            }
-            firstSource = firstSource.replaceAll(".*http[s]?://", "").replaceAll("/.*", "");
+            firstSource = firstSource.replaceAll(".*http[s]?://", "").replaceAll("/.*", "").replaceAll("www.", "");
-            sessionSource = sessionSource.replaceAll(".*http[s]?://", "").replaceAll("/.*", "");
+            sessionSource = sessionSource.replaceAll(".*http[s]?://", "").replaceAll("/.*", "").replaceAll("www.", "");
             List<Long> itemIds = null;
-            if (!result.hasProperty(ITEMIDS_FILED)) {
+            if (!result.hasProperty(ITEMIDS_FIELD)) {
                 continue;
+            }
             try {
-                itemIds = (List<Long>)result.getProperty(ITEMIDS_FILED);
+                itemIds = (List<Long>)result.getProperty(ITEMIDS_FIELD);
             } catch (ClassCastException e) {
                 try {
                     itemIds = new ArrayList<Long>();
-                    itemIds.add((Long)result.getProperty(ITEMIDS_FILED));
+                    itemIds.add((Long)result.getProperty(ITEMIDS_FIELD));
+                }
                 catch(Exception ex) {
                     log(e.getMessage());
+                }
+            }

Subversion Repositories SmartDukaan

(root)/trunk/GoogleAppDatalog/src/main/java/in/shop2020/web/DailyProductWithSourceAggregatorServlet.java – Rev 3444 → 3488