WebSVN – SmartDukaan – Diff – //trunk/ContentStore/src/main/java/in/shop2020/util/ContentGenerationUtility.java

 import in.shop2020.metamodel.core.EntityStatus;
 import in.shop2020.metamodel.util.CreationUtils;
 import in.shop2020.metamodel.util.ExpandedEntity;
 import in.shop2020.model.v1.catalog.CatalogService.Client;
 import in.shop2020.model.v1.catalog.Item;
-import in.shop2020.model.v1.catalog.ItemShippingInfo;
 import in.shop2020.model.v1.catalog.Source;
 import in.shop2020.model.v1.catalog.status;
 import in.shop2020.thrift.clients.CatalogClient;
-import in.shop2020.ui.util.CatalogUploderToGAE;
 import in.shop2020.ui.util.ComparisonStatsFetcher;
 import in.shop2020.ui.util.NewVUI;
 import in.shop2020.ui.util.PriceInsertor;
 import in.shop2020.ui.util.SpecialPageJSONConvertor;
 import in.shop2020.utils.ConfigClientKeys;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import org.apache.commons.cli.ParseException;
 import org.apache.commons.cli.PosixParser;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.thrift.transport.TTransportException;
 import com.google.gson.Gson;
 import com.google.gson.reflect.TypeToken;
 public class ContentGenerationUtility {
     private static String [] 	  DOMAINPATHS 			      = Utils.DOMAIN_NAMES_FOR_CONTENT_GENERATION.split(";");
     private Date 				  timeStamp			  		  = null;
     private CommandLine           cmd                         = null;
     private Map<Long, List<Item>> entityIdItemMap             = new LinkedHashMap<Long, List<Item>>();
-    private List<Long> 			  allValidEntityIds 			  = null;
     private Long                  lastGenerationTime          = 0l;
     private Map<Long, Entity>     entities;
     private List<Item>            items;
     private List<Source>          sources;
     private CatalogClient         csc;
+     *
      * @throws Exception
      */
     private void updatePrices() throws Exception {
         lastGenerationTime = new Long(0);
+        List<Long> activeItems = new ArrayList<Long>();
         if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {
             items = client.getItemsByCatalogId(Long.parseLong(ENTITY_ID));
             Iterator<Item> it = items.iterator();
             while(it.hasNext()){
+            	Item ite = it.next();
-            	status st = it.next().getItemStatus();
+            	status st = ite.getItemStatus();
             	if(!(st.equals(status.ACTIVE) || st.equals(status.PAUSED) ||
-            				st.equals(status.COMING_SOON) || st.equals(status.PHASED_OUT))){
+            				st.equals(status.COMING_SOON))){
             		it.remove();
+            	}
+            	if(st.equals(status.ACTIVE)){
+            		activeItems.add(ite.getId());
+            	}
+            }
-            try {
-            	//Generate prices and availability data for amazon
-            	AmazonSCDataGenerator.generatePricesAndAvailability(items);
-            } catch (Exception e) {
-            	log.info("Could not generate Amazon prices and availability", e);
-            }
             //ProductListGenerator.updatePriceForEntity(Long.parseLong(ENTITY_ID), items.get(0).getSellingPrice(), items.get(0).getMrp());
         } else {
         	log.info("Before getting active items.");
             items = client.getAllItemsByStatus(status.ACTIVE);
+            for (Item item : items) {
+            	activeItems.add(item.getId());
+            }
             log.info("Before getting coming items.");
             items.addAll(client.getAllItemsByStatus(status.COMING_SOON));
             log.info("Before getting paused items.");
             items.addAll(client.getAllItemsByStatus(status.PAUSED));
             // Clean up the data from the solr directories.
         if (GENERATION_TYPE.equals(GENERATION_TYPE_INCREMENTAL)) {
+        }
         // Populate the entityIdIemMap
         populateEntityIdItemMap();
-        List<Long> inStockEntities = null;
         // Generate partners and json objects for phones only
         if (!GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {
-        	ProductListGenerator generator = new ProductListGenerator(entityIdItemMap);
-        	log.info("Before thinkdigit feed.");
-        	generator.generateThinkDigitFeed();
-        	inStockEntities = generator.getInStockCatalogItemIds();
-        	log.info("Before product list js.");
-        	generator.generateProductListJavascript();
         	try	{
-        		log.info("Before product list xml.");
-            	generator.generateProductsListXML();
+	        	ProductListGenerator generator = new ProductListGenerator(entityIdItemMap);
-            	log.info("Before product accessories xml.");
-            	generator.generateAccessoriesXML();
-            	log.info("Before product camera xml.");
+	        	log.info("Before auto suggest json");
-            	generator.generateCamerasXML();
+	        	synonymTitlesExporter();
-            	log.info("Before product display ads.");
+	        	log.info("Before product list js.");
-        		//generator.generateProductXMLForDisplayAds();
+	        	generator.generateProductListJavascript();
         	} catch (Exception e) {
         		e.printStackTrace();
+        	}
+        }
         PriceInsertor priceInserter = new PriceInsertor();
         Map<Long,List<String>> entityTags = client.getAllEntityTags();
         Map<Long, Integer> popularityMap = getPolularityMap();
+        CatalogClient prod_csc = new CatalogClient(ConfigClientKeys.catalog_service_server_host_prod.toString(), ConfigClientKeys.catalog_service_server_port.toString());
+        Client prod_client = prod_csc.getClient();
+        Map<Long,Boolean> itemsStatus = prod_client.getItemsStatus(activeItems);
         for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {
             long entityId = entry.getKey();
             List<Item> items = entry.getValue();
-	            // TODO Domain name and destination directory should be read from
-	            // properties file
             double minPrice = 0d;
+            //Evaluating availability
-            String availability = "In Stock";
+            String availability = "Out of Stock";
-            if (inStockEntities != null ) {
+            for(Item i: items){
-            	if (!inStockEntities.contains(entry.getKey())) {
+            	if(itemsStatus.containsKey(i.getId())) {
+            		if(itemsStatus.get(i.getId())){
+            			availability = "In Stock";
+            			break;
+            		}
+            	}else if(i.getItemStatus().equals(status.ACTIVE)){
-            		availability = "Out of Stock";
+            		availability = "In Stock";
+            		break;
+            	}
-            } else {
-            	availability = getAvaialability(entry.getValue());
+            }
             StringBuilder priceString = new StringBuilder();
             StringBuilder availabilityString = new StringBuilder();
             boolean domainOnce = true;
             boolean sourceOnce = true;
             priceInserter.insertPriceInSolrData(entityId,
                     priceString.toString(), availabilityString.toString());
+        }
         priceInserter.copySolrSchemaFiles();
-        synonymTitlesExporter();
+    }
-    private boolean isStoreItem(List<Item> items2) {
-		for (Item item : items2){
-			if(item.isActiveOnStore()) {
-				return true;
-			}
-		}
-		return false;
-	}
 	private Map<Long, Integer> getPolularityMap() {
 		try {
 			Reader reader = new FileReader(Utils.EXPORT_PATH + Utils.POPULARITY_JSON);
 			return new Gson().fromJson(reader, new TypeToken<Map<Long, Integer>>() {}.getType());
 		} catch (FileNotFoundException e) {
 			return new HashMap<Long, Integer>();
+		}
+	}
-	private String getAvaialability(List<Item> value) {
-    	boolean isActive = true;
-    	try {
-			Client catalogClientProd = new CatalogClient(ConfigClientKeys.catalog_service_server_host_prod.toString(), ConfigClientKeys.catalog_service_server_port.toString()).getClient();
-			for (Item item : value ) {
-				if (item.getItemStatus().equals(status.ACTIVE)) {
-					if(item.isRisky()){
-	    				try {
-	    					ItemShippingInfo isi = catalogClientProd.isActive(item.getId());
-	    					isActive = isi.isIsActive();
-	    				} catch (Exception e) {
-	    					e.printStackTrace();
-	    					isActive = true;
-	    				}
-	    			}
-				}
-				if(isActive) break;
-			}
-		} catch (TTransportException e) {
-			// TODO Auto-generated catch block
-			e.printStackTrace();
-		}
-		return isActive ? "In Stock" : "Out of Stock";
-	}
 	/**
      * Generates content for the specified entity embedding links to the
      * specified domain name.
+     *
      * The method will not generate content if one of the following conditions
      */
     private void generateContent() throws Exception {
         if (GENERATION_TYPE.equals(GENERATION_TYPE_ALL)) {
             entities = CreationUtils.getEntities();
             lastGenerationTime = new Long(0);
+            items = client.getAllItemsByStatus(status.CONTENT_COMPLETE);
+            items.addAll(client.getAllItemsByStatus(status.COMING_SOON));
+            items.addAll(client.getAllItemsByStatus(status.ACTIVE));
+            items.addAll(client.getAllItemsByStatus(status.PAUSED));
         } else if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {
             entities = new HashMap<Long, Entity>();
             entities.put(Long.parseLong(ENTITY_ID),
                     CreationUtils.getEntity(Long.parseLong(ENTITY_ID)));
+            items = client.getItemsByCatalogId(Long.parseLong(ENTITY_ID));
+            Iterator<Item> ite = items.iterator();
+            while(ite.hasNext()) {
+            	Item i = ite.next();
+            	if(!(i.getItemStatus().equals(status.ACTIVE) || i.getItemStatus().equals(status.PAUSED)
+            	|| i.getItemStatus().equals(status.CONTENT_COMPLETE) || i.getItemStatus().equals(status.COMING_SOON))){
+            		ite.remove();
+            	}
+            }
             lastGenerationTime = new Long(0);
         } else {
         	entities = CreationUtils.getEntities();
             //  When we read lastGenerationTime from database
             //  then only we should mark the
             log.info("lastGenerationTime: " + lastGenerationTime);
             if (lastGenerationTime == null) {
                 lastGenerationTime = new Long(0);
+            }
+            items = client.getAllItemsByStatus(status.CONTENT_COMPLETE);
+            items.addAll(client.getAllItemsByStatus(status.COMING_SOON));
+        }
         // Filter invalid entities here
         List<Entity> validEntities = new ArrayList<Entity>();
+        List<Long> validEntityIds = new ArrayList<Long>();
         for (long entityID : entities.keySet()) {
             if (isValidEntity(entities.get(entityID))) {
                 validEntities.add(entities.get(entityID));
+                validEntityIds.add(entityID);
+            }
+        }
         // Calculate comparison scores
         log.info("Calculating comparison scores");
         NewCMP cmp = new NewCMP(validEntities);
         Map<Long, Map<Long, Double>> slideScoresByEntity = cmp.getSlideScores();
         CreationUtils.storeSlideScores(slideScoresByEntity);
         // Fetch comparison statistics everyday and store them in BDB
+        //This might be remove.
         log.info("Fetching comparison statistics");
         ComparisonStatsFetcher csf = new ComparisonStatsFetcher();
         csf.fetchAndStoreComparisonStats();
-        // Upload catalog to Google App Engine.
-        if (GENERATION_TYPE.equals(GENERATION_TYPE_ALL)) {
-            log.info("Uploading Catalog to Google app engine");
-            List<Item> allItems = client.getAllItems(false);
-            allItems.addAll(client.getAllItems(true));
-            CatalogUploderToGAE catalogUploaderToGAE = new CatalogUploderToGAE();
-            catalogUploaderToGAE.uploadItems(allItems);
-        }
-        items = client.getAllItemsByStatus(status.ACTIVE);
-        items.addAll(client.getAllItemsByStatus(status.PAUSED));
-        items.addAll(client.getAllItemsByStatus(status.CONTENT_COMPLETE));
-        items.addAll(client.getAllItemsByStatus(status.COMING_SOON));
         populateEntityIdItemMap();
-        //FIXME Avoiding the finding of accesories, as list of categories for which we need to find accessories is hardocoded in code.
-        // We need to make that configurable. Also creating ticket to improve it.
-        try{
-	        log.info("Finding accessories");
-	        AccessoriesFinder af = new AccessoriesFinder(new HashSet<Long>(allValidEntityIds));
-	        Map<Long, Map<Long, List<Long>>> relatedAccessories = af.findAccessories();
-	        CreationUtils.storeRelatedAccessories(relatedAccessories);
-        }catch (Exception e) {
-        	log.error("Error while generating accessories" + e);
-		}
         log.info("Writing JSON file for special pages");
         SpecialPageJSONConvertor bjc = new SpecialPageJSONConvertor();
         bjc.writeToJSONFile(new File(Utils.EXPORT_JAVASCRIPT_CONTENT_PATH
                 + "special-pages.json"));
         for (Entity entity : validEntities) {
             log.info("Processing Entityid: " + entity.getID());
             vui.generateContentForOneEntity(entity, Utils.EXPORT_VELOCITY_PATH);
+        }
-        // Generate synonyms list. This will be used in PriceComparisonTool to
-        // resolve the product names.
-        log.info("Generating synonyms");
-        SynonymExporter sx = new SynonymExporter();
-        sx.storeSynonyms(validEntities);
-        List<Entity> allValidEntities;
-        if (GENERATION_TYPE.equals(GENERATION_TYPE_ALL))	{
-        	allValidEntities = validEntities;
-        } else	{
-        	allValidEntities = new ArrayList<Entity>();
-			for (Long entityId : allValidEntityIds)	{
-				allValidEntities.add(CreationUtils.getEntity(entityId));
-			}
-        }
-        log.info("Generating HTML for Site Index");
-        ProductIndexGenerator indexGenerator = new ProductIndexGenerator(allValidEntities);
-        indexGenerator.generate();
-        log.info("Generating HTML for Site for Product Documents");
-        ProductDocumentsGenerator asGenerator = new ProductDocumentsGenerator(allValidEntities);
-        asGenerator.generate();
-        log.info("Generating HTML for Accessories Compatibility Index");
-        CompatibleAccessoriesIndexGenerator generator = new CompatibleAccessoriesIndexGenerator(allValidEntities);
-        generator.generate();
-        log.info("Generating HTML for Most Frequently searched keywords");
-        MostFrequentlySearchedKeywords mfsk = new MostFrequentlySearchedKeywords();
-        mfsk.generate();
-        log.info("Generating HTML for Most Compared Index");
-        MostComparedIndexGenerator mostCompGenerator = new MostComparedIndexGenerator(allValidEntityIds);
-        mostCompGenerator.generate();
-        log.info("Generating XML for Mobile Site XML feed");
-        MobileSiteDataXMLGenerator mSiteXMLGenerator = new MobileSiteDataXMLGenerator(allValidEntities);
-        mSiteXMLGenerator.generate();
         if (newLastGenerationTime != 0) {
             CreationUtils.storeLastContentGenerationTime(newLastGenerationTime);
+        }
         log.info("Generating Solr files");
         NewIR ir = new NewIR(validEntities);
         ir.exportIRData();
         // ir.transformIrDataXMLtoSolrXML();
+        if(!GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {
-        ir.exportIRMetaData();
+	        ir.exportIRMetaData();
-        ir.transformIrMetaDataXMLtoSolrSchemaXML();
+	        ir.transformIrMetaDataXMLtoSolrSchemaXML();
-        ir.transformIrMetaDataXMLtoSolrCatchAllXML();
+	        ir.transformIrMetaDataXMLtoSolrCatchAllXML();
+        }
         for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {
             List<Item> items = entry.getValue();
             for (Item item : items) {
-                if (item.getItemStatus() == status.CONTENT_COMPLETE || item.getItemStatus() == status.COMING_SOON) {
+                if ( (item.getItemStatus() == status.CONTENT_COMPLETE && validEntityIds.contains(item.getCatalogItemId()))
+                		|| item.getItemStatus() == status.COMING_SOON) {
                 	if(item.getStartDate() <= new Date().getTime() + ONE_DAY){
                 		item.setItemStatus(status.ACTIVE);
                 		item.setStatus_description("This item is active");
                 	} else {
                 		item.setItemStatus(status.COMING_SOON);
                     client.updateItem(item);
+            	}
+            }
+        }
         sendAlertToCategoryTeam(alertItems);
-        try {
-            //generate products list that is to be uploaded in Amazon.
-        	AmazonSCDataGenerator ascdGenerator = new AmazonSCDataGenerator(validEntities, GENERATION_TYPE);
-            ascdGenerator.generateSCProdData();
-        } catch (Exception e) {
-        	e.printStackTrace();
-            log.info("Could not generate Amazon data", e);
-        }
+    }
     private void sendAlertToCategoryTeam(List<Item> items) {
     	if(items!=null && items.size()!=0){
 			GmailUtils util = new GmailUtils();
+            }
             if (!isValidEntity) {
                 removeEntities.add(entityId);
+            }
+        }
-        //Simply assign allValidEntityIds to a class variable as these need to be used where all valid entites
-        //are needed.
-        allValidEntityIds = new ArrayList<Long>(entityIdItemMap.keySet());
         for (Long entityId : removeEntities) {
             entityIdItemMap.remove(entityId);
+        }
         Utils.info("Final valid entities to be processed: " + entityIdItemMap.size());

Subversion Repositories SmartDukaan

(root)//trunk/ContentStore/src/main/java/in/shop2020/util/ContentGenerationUtility.java – Rev 7336 → 7506