Subversion Repositories SmartDukaan

package in.shop2020.util;

import in.shop2020.metamodel.core.Entity;

import in.shop2020.metamodel.core.EntityState;

import in.shop2020.metamodel.core.EntityStatus;

import in.shop2020.metamodel.util.CreationUtils;

import in.shop2020.metamodel.util.ExpandedEntity;

import in.shop2020.model.v1.catalog.CatalogService.Client;

import in.shop2020.model.v1.catalog.Item;

import in.shop2020.model.v1.catalog.ItemShippingInfo;

import in.shop2020.model.v1.catalog.Source;

import in.shop2020.model.v1.catalog.status;

import in.shop2020.thrift.clients.CatalogClient;

import in.shop2020.ui.util.CatalogUploderToGAE;

import in.shop2020.ui.util.ComparisonStatsFetcher;

import in.shop2020.ui.util.NewVUI;

import in.shop2020.ui.util.PriceInsertor;

import in.shop2020.ui.util.SpecialPageJSONConvertor;

import in.shop2020.utils.ConfigClientKeys;

import in.shop2020.utils.GmailUtils;

import java.io.File;

import java.io.IOException;

import java.text.SimpleDateFormat;

import java.util.ArrayList;

import java.util.Calendar;

import java.util.Date;

import java.util.HashMap;

import java.util.HashSet;

import java.util.Iterator;

import java.util.LinkedHashMap;

import java.util.List;

import java.util.Map;

import javax.mail.MessagingException;

import org.apache.commons.cli.CommandLine;

import org.apache.commons.cli.CommandLineParser;

import org.apache.commons.cli.HelpFormatter;

import org.apache.commons.cli.Options;

import org.apache.commons.cli.ParseException;

import org.apache.commons.cli.PosixParser;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.thrift.transport.TTransportException;

import com.google.gson.Gson;

public class ContentGenerationUtility {

    private static final String   UPDATE_TYPE_CATALOG         = "CATALOG";

    private static final String   UPDATE_TYPE_CONTENT         = "CONTENT";

    private static final String   COMMAND_LINE                = "java ContentGenerationUtility.class -t { ALL | INCREMENTAL | ONE } -s { yyyy-MM-dd-HH-mm-ss } -u { CONTENT | CATALOG } -e {EntityId} ";

    private static Log            log                         = LogFactory

                                                                      .getLog(ContentGenerationUtility.class);

    private static long 			ONE_DAY					  = 24*60*60*1000;  	//milliseconds in a day						 	

    // Commandline options

    private static Options        options                     = null;

    private static final String   GENERATION_TYPE_INCREMENTAL = "INCREMENTAL";

    private static final String   GENERATION_TYPE_ALL         = "ALL";

    private static final String   GENERATION_TYPE_ONE         = "ONE";

    private static final String   UPDATE_TYPE_OPTION          = "u";

    private static final String   GENERATION_TYPE_OPTION      = "t";

    private static final String   ENTITY_ID_OPTION            = "e";

    private static final String   TIMESTAMP_OPTION            = "s";

    // Default values of cmdline options

    private static String         UPDATE_TYPE                 = UPDATE_TYPE_CONTENT;

    private static String         GENERATION_TYPE             = GENERATION_TYPE_INCREMENTAL;

    private static String         ENTITY_ID                   = "ALL";

    private static String [] 	  DOMAINPATHS 			      = Utils.DOMAIN_NAMES_FOR_CONTENT_GENERATION.split(";");

    private Date 				  timeStamp			  		  = null;

    private CommandLine           cmd                         = null;

    private Map<Long, List<Item>> entityIdItemMap             = new LinkedHashMap<Long, List<Item>>();

    private List<Long> 			  allValidEntityIds 			  = null;

    private Long                  lastGenerationTime          = 0l;

    private Map<Long, Entity>     entities;

    private List<Item>            items;

    private List<Source>          sources;

    private CatalogClient         csc;

    private Client                client;

    private List<Item>			  alertItems;	

    private long                  newLastGenerationTime;

    static {

        options = new Options();

        options.addOption(GENERATION_TYPE_OPTION, true, "Generation type");

        options.addOption(UPDATE_TYPE_OPTION, true, "Default is : "

                + UPDATE_TYPE);

        options.addOption(ENTITY_ID_OPTION, true, "all entities " + ENTITY_ID

                + " by default");

        options.addOption(TIMESTAMP_OPTION, true, "Manual timestamp");

    }

    public ContentGenerationUtility() throws Exception {

        csc = new CatalogClient();

        client = csc.getClient();

        sources = client.getAllSources();

        alertItems = new ArrayList<Item>();	

    }

    /**

     * @param args

     * @throws Exception

     */

    public static void main(String[] args) throws Exception {

        ContentGenerationUtility cgu = new ContentGenerationUtility();

        // Load arguments

        cgu.loadArgs(args);

        // Call method based on arguments

        cgu.callMethod();

    }

    /**

     * Validate and set command line arguments. Exit after printing usage if

     * anything is astray

     * 

     * @param args

     *            String[] args as featured in public static void main()

     */

    private void loadArgs(String[] args) {

        CommandLineParser parser = new PosixParser();

        try {

            cmd = parser.parse(options, args);

        } catch (ParseException e) {

            log.error("Error parsing arguments", e);

            System.exit(1);

        }

        // Check for mandatory args

        if (!(cmd.hasOption(GENERATION_TYPE_OPTION) && cmd

                .hasOption(UPDATE_TYPE_OPTION))) {

            HelpFormatter formatter = new HelpFormatter();

            formatter.printHelp(COMMAND_LINE, options);

            System.exit(1);

        }

        GENERATION_TYPE = cmd.getOptionValue(GENERATION_TYPE_OPTION);

        UPDATE_TYPE = cmd.getOptionValue(UPDATE_TYPE_OPTION);

        // Look for optional args.

        if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {

            if (cmd.hasOption(ENTITY_ID_OPTION)) {

                ENTITY_ID = cmd.getOptionValue(ENTITY_ID_OPTION);

            } else {

                HelpFormatter formatter = new HelpFormatter();

                formatter.printHelp(COMMAND_LINE, options);

                System.exit(1);

            }

        }

        if (GENERATION_TYPE_INCREMENTAL.equals(GENERATION_TYPE))

        	if (cmd.hasOption(TIMESTAMP_OPTION)) {

        		SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");

        		try {

        		    timeStamp = df.parse(cmd.getOptionValue(TIMESTAMP_OPTION));

        		} catch(Exception e) {

        			HelpFormatter formatter = new HelpFormatter();

                    formatter.printHelp(COMMAND_LINE, options);

                    System.exit(1);

        		}

        	}

    }

    /**

     * Call method based on arguments

     * 

     * @throws Exception

     */

    private void callMethod() {

        boolean isSuccess = false;

        String logfile = "/tmp/content-from-cms.log";

        if (UPDATE_TYPE.equals(UPDATE_TYPE_CONTENT)) {

            logfile = "/tmp/content-from-cms.log";

            try {

                this.generateContent();

                isSuccess = true;

            } catch (Exception e) {

                log.error("Error generating content", e);

            }

        }

        if (UPDATE_TYPE.equals(UPDATE_TYPE_CATALOG)) {

            logfile = "/tmp/content-from-catalog.log";

            try {

                this.updatePrices();

                isSuccess = true;

            } catch (Exception e) {

                log.error("Error updating prices", e);

            }

        }

        GmailUtils gm = new GmailUtils();

        String[] sendTo = { "rajveer.singh@shop2020.in", "mandeep.dhir@shop2020.in", "pankaj.kankar@shop2020.in", "anupam.singh@shop2020.in", "amit.gupta@shop2020.in" };

        try {

            gm.sendSSLMessage(sendTo, "Content Generation Successful ? : "

                    + isSuccess, "Content generation completed at time : "

                    + Calendar.getInstance().getTime().toString(),

                    "build@shop2020.in", "cafe@nes", logfile);

        } catch (MessagingException e) {

            log.error("Could not send status mail", e);

        }

    }

    public boolean cleanDir(File dir, boolean deleteSelf) {

        if (dir.isDirectory()) {

            String[] children = dir.list();

            for (int i = 0; i < children.length; i++) {

                boolean success = cleanDir(new File(dir, children[i]), true);

                if (!success) {

                    return false;

                }

            }

        }

        // The directory is now empty so delete it

        if (deleteSelf) {

            return dir.delete();

        }

        return true;

    }

    private void removeOldResources() throws IOException {

        File f = new File(Utils.EXPORT_SOLR_PATH);

        if (f.exists()) {

            cleanDir(f, false);

        }

        for(String domainPath : DOMAINPATHS){

        	String pathName = domainPath.split("\\.")[0].split(":")[0];

        	File f1 = new File(Utils.EXPORT_PATH + "html/entities-" +  pathName);

        	if (f1.exists()) {

        		cleanDir(f1, false);

        	}else {

        		f1.mkdir();

        	}

        }

    }

    /**

     * Update the prices in the generated content

     * 

     * @throws Exception

     */

    private void updatePrices() throws Exception {

        lastGenerationTime = new Long(0);

        if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {

            items = client.getItemsByCatalogId(Long.parseLong(ENTITY_ID));

            Iterator<Item> it = items.iterator();

            while(it.hasNext()){

            	status st = it.next().getItemStatus();

            	if(!(st.equals(status.ACTIVE) || st.equals(status.PAUSED) || 

            				st.equals(status.COMING_SOON) || st.equals(status.PHASED_OUT))){

            		it.remove();

            	}

            }

            try {

            	//Generate prices and availability data for amazon

            	AmazonSCDataGenerator.generatePricesAndAvailability(items);

            } catch (Exception e) {

            	log.info("Could not generate Amazon prices and availability", e);

            }

            //ProductListGenerator.updatePriceForEntity(Long.parseLong(ENTITY_ID), items.get(0).getSellingPrice(), items.get(0).getMrp());

        } else {

        	log.info("Before getting active items.");

            items = client.getAllItemsByStatus(status.ACTIVE);

            log.info("Before getting coming items.");

            items.addAll(client.getAllItemsByStatus(status.COMING_SOON));

            log.info("Before getting paused items.");

            items.addAll(client.getAllItemsByStatus(status.PAUSED));

            // Clean up the data from the solr directories.

            log.info("Before removing old resources.");

            removeOldResources();

        }

        // this still needs to be evolved. Must not be used.

        if (GENERATION_TYPE.equals(GENERATION_TYPE_INCREMENTAL)) {

        }

        // Populate the entityIdIemMap

        populateEntityIdItemMap();

        List<Long> inStockEntities = null;

        // Generate partners and json objects for phones only

        if (!GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {

        	ProductListGenerator generator = new ProductListGenerator(entityIdItemMap);

        	log.info("Before thinkdigit feed.");

        	generator.generateThinkDigitFeed();

        	inStockEntities = generator.getInStockCatalogItemIds();

        	log.info("Before product list js.");

        	generator.generateProductListJavascript();

        	try	{

        		log.info("Before product list xml.");

            	generator.generateProductsListXML();

            	log.info("Before product accessories xml.");

            	generator.generateAccessoriesXML();

            	log.info("Before product camera xml.");

            	generator.generateCamerasXML();

            	log.info("Before product display ads.");

        		//generator.generateProductXMLForDisplayAds();

        	} catch (Exception e) {

        		e.printStackTrace();

        	}

        }

        PriceInsertor priceInserter = new PriceInsertor();

        for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {

            long entityId = entry.getKey();

            List<Item> items = entry.getValue();

	            // TODO Domain name and destination directory should be read from

	            // properties file

            double minPrice = 0d;

            String availability = "In Stock";

            if (inStockEntities != null ) {

            	if (!inStockEntities.contains(entry.getKey())) {

            		availability = "Out of Stock";

            	}

            } else {

            	availability = getAvaialability(entry.getValue());

            }

            StringBuilder priceString = new StringBuilder();

            StringBuilder availabilityString = new StringBuilder();

            boolean domainOnce = true;

            boolean sourceOnce = true;

            for(String domainPath : DOMAINPATHS){

            	String domainName = domainPath;

            	String pathName = domainPath.split("\\.")[0].split(":")[0];

            	minPrice = priceInserter.insertPriceInHtml(items, entityId,

            			domainName, Utils.EXPORT_PATH + "html/entities-" +  pathName + "/", null);

            	if(domainOnce){

            		priceString.append("<field name=\"F_50002\">" + minPrice + "</field>");

            		availabilityString.append("<field name=\"F_50028\">" + availability + "</field>");

            		domainOnce = false;

            	}

            	if(sources != null){

            		for (Source source : sources) {

                        minPrice = priceInserter.insertPriceInHtml(items, entityId,

                                domainName, Utils.EXPORT_PATH + "html/entities-" + pathName + "/",

                                source);

                        if(sourceOnce){

                        	priceString.append("<field name=\"F_50002_"

                                + source.getId() + "\">" + minPrice + "</field>");

                        	availabilityString.append("<field name=\"F_50028_"

                        			+ source.getId() + "\">" + availability + "</field>");

                        }

            		}

            		sourceOnce = false;

            	}

            }

            priceInserter.insertPriceInSolrData(entityId,

                    priceString.toString(), availabilityString.toString());

        }

        priceInserter.copySolrSchemaFiles();

        synonymTitlesExporter();

    }

    private String getAvaialability(List<Item> value) {

    	boolean isActive = true;

    	try {

			Client catalogClientProd = new CatalogClient(ConfigClientKeys.catalog_service_server_host_prod.toString(), ConfigClientKeys.catalog_service_server_port.toString()).getClient();

			for (Item item : value ) {

				if (item.getItemStatus().equals(status.ACTIVE)) {

					if(item.isRisky()){

	    				try {

	    					ItemShippingInfo isi = catalogClientProd.isActive(item.getId());

	    					isActive = isi.isIsActive();

	    				} catch (Exception e) {

	    					e.printStackTrace();

	    					isActive = true;

	    				}

	    			}

				}

				if(isActive) break;

			}

		} catch (TTransportException e) {

			// TODO Auto-generated catch block

			e.printStackTrace();

		}

		return isActive ? "In Stock" : "Out of Stock";

	}

	/**

     * Generates content for the specified entity embedding links to the

     * specified domain name.

     * 

     * The method will not generate content if one of the following conditions

     * is met:

     * <ol>

     * <li>The entity is not ready.

     * <li>The category has not been updated yet. (Set to -1).

     * <li>The content has not been updated.

     * </ol>

     * 

     * @throws

     */

    private void generateContent() throws Exception {

        if (GENERATION_TYPE.equals(GENERATION_TYPE_ALL)) {

            entities = CreationUtils.getEntities();

            lastGenerationTime = new Long(0);

        } else if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {

            entities = new HashMap<Long, Entity>();

            entities.put(Long.parseLong(ENTITY_ID),

                    CreationUtils.getEntity(Long.parseLong(ENTITY_ID)));

            lastGenerationTime = new Long(0);

        } else {

        	entities = CreationUtils.getEntities();

            //  When we read lastGenerationTime from database

            //  then only we should mark the 

            //	current time as newLastGenerationTime

            if(timeStamp == null) {

            	newLastGenerationTime = new Date().getTime();

            	lastGenerationTime = CreationUtils.getLastContentGenerationTime();

            } else {

            	lastGenerationTime = timeStamp.getTime();

            }

            log.info("lastGenerationTime: " + lastGenerationTime);

            if (lastGenerationTime == null) {

                lastGenerationTime = new Long(0);

            }

        } 

        // Filter invalid entities here

        List<Entity> validEntities = new ArrayList<Entity>();

        for (long entityID : entities.keySet()) {

            if (isValidEntity(entities.get(entityID))) {

                validEntities.add(entities.get(entityID));

            }

        }

        // Calculate comparison scores

        log.info("Calculating comparison scores");

        NewCMP cmp = new NewCMP(validEntities);

        Map<Long, Map<Long, Double>> slideScoresByEntity = cmp.getSlideScores();

        CreationUtils.storeSlideScores(slideScoresByEntity);

        // Fetch comparison statistics everyday and store them in BDB

        log.info("Fetching comparison statistics");

        ComparisonStatsFetcher csf = new ComparisonStatsFetcher();

        csf.fetchAndStoreComparisonStats();

        // Upload catalog to Google App Engine.

        if (GENERATION_TYPE.equals(GENERATION_TYPE_ALL)) {

            log.info("Uploading Catalog to Google app engine");

            List<Item> allItems = client.getAllItems(false);

            allItems.addAll(client.getAllItems(true));

            CatalogUploderToGAE catalogUploaderToGAE = new CatalogUploderToGAE();

            catalogUploaderToGAE.uploadItems(allItems);

        }

        items = client.getAllItemsByStatus(status.ACTIVE);

        items.addAll(client.getAllItemsByStatus(status.PAUSED));

        items.addAll(client.getAllItemsByStatus(status.CONTENT_COMPLETE));

        items.addAll(client.getAllItemsByStatus(status.COMING_SOON));

        populateEntityIdItemMap();

        //FIXME Avoiding the finding of accesories, as list of categories for which we need to find accessories is hardocoded in code. 

        // We need to make that configurable. Also creating ticket to improve it.

        try{

	        log.info("Finding accessories");

	        AccessoriesFinder af = new AccessoriesFinder(new HashSet<Long>(allValidEntityIds));

	        Map<Long, Map<Long, List<Long>>> relatedAccessories = af.findAccessories();

	        CreationUtils.storeRelatedAccessories(relatedAccessories);

        }catch (Exception e) {

        	log.error("Error while generating accessories" + e);

		}

        log.info("Writing JSON file for special pages");

        SpecialPageJSONConvertor bjc = new SpecialPageJSONConvertor();

        bjc.writeToJSONFile(new File(Utils.EXPORT_JAVASCRIPT_CONTENT_PATH

                + "special-pages.json"));

        log.info("Generating velocity templates, images, documents etc.");

        NewVUI vui = new NewVUI(lastGenerationTime);

        for (Entity entity : validEntities) {

            log.info("Processing Entityid: " + entity.getID());

            vui.generateContentForOneEntity(entity, Utils.EXPORT_VELOCITY_PATH);

        }

        // Generate synonyms list. This will be used in PriceComparisonTool to

        // resolve the product names.

        log.info("Generating synonyms");

        SynonymExporter sx = new SynonymExporter();

        sx.storeSynonyms(validEntities);

        List<Entity> allValidEntities;

        if (GENERATION_TYPE.equals(GENERATION_TYPE_ALL))	{

        	allValidEntities = validEntities;

        } else	{

        	allValidEntities = new ArrayList<Entity>();

			for (Long entityId : allValidEntityIds)	{

				allValidEntities.add(CreationUtils.getEntity(entityId));

			}

        }

        log.info("Generating HTML for Site Index");

        ProductIndexGenerator indexGenerator = new ProductIndexGenerator(allValidEntities);

        indexGenerator.generate();

        log.info("Generating HTML for Site for Product Documents");

        ProductDocumentsGenerator asGenerator = new ProductDocumentsGenerator(allValidEntities);

        asGenerator.generate();

        log.info("Generating HTML for Accessories Compatibility Index");

        CompatibleAccessoriesIndexGenerator generator = new CompatibleAccessoriesIndexGenerator(allValidEntities);

        generator.generate();

        log.info("Generating HTML for Most Frequently searched keywords");

        MostFrequentlySearchedKeywords mfsk = new MostFrequentlySearchedKeywords();

        mfsk.generate();

        log.info("Generating HTML for Most Compared Index");

        MostComparedIndexGenerator mostCompGenerator = new MostComparedIndexGenerator(allValidEntityIds);

        mostCompGenerator.generate();

        log.info("Generating XML for Mobile Site XML feed");

        MobileSiteDataXMLGenerator mSiteXMLGenerator = new MobileSiteDataXMLGenerator(allValidEntities);

        mSiteXMLGenerator.generate();

        if (newLastGenerationTime != 0) {

            CreationUtils.storeLastContentGenerationTime(newLastGenerationTime);

        }

        log.info("Generating Solr files");

        NewIR ir = new NewIR(validEntities);

        ir.exportIRData();

        // ir.transformIrDataXMLtoSolrXML();

        ir.exportIRMetaData();

        ir.transformIrMetaDataXMLtoSolrSchemaXML();

        ir.transformIrMetaDataXMLtoSolrCatchAllXML();

        for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {

            List<Item> items = entry.getValue();

            for (Item item : items) {

                if (item.getItemStatus() == status.CONTENT_COMPLETE || item.getItemStatus() == status.COMING_SOON) {

                	if(item.getStartDate() <= new Date().getTime() + ONE_DAY){

                		item.setItemStatus(status.ACTIVE);

                		item.setStatus_description("This item is active");

                	} else {

                		item.setItemStatus(status.COMING_SOON);

                		String productName = getProductName(item);

                		String statusDescription = productName + " is coming soon.";

                		if(item.getExpectedArrivalDate()>new Date().getTime() + ONE_DAY){

                			statusDescription = productName + " will be available by " 

                			+ new SimpleDateFormat("dd/MM/yy").format(new Date(item.getExpectedArrivalDate()));

                		}

                		//Send alert to Category team one day before expected arrival date

                		//So they may change the expected arrival date if they want to.

                		if(item.getExpectedArrivalDate() < new Date().getTime() + 2*ONE_DAY && 

                				item.getExpectedArrivalDate() > new Date().getTime() + ONE_DAY) {

                				alertItems.add(item);

                		}

                		item.setStatus_description(statusDescription);

                	}

                    client.updateItem(item);

            	}

            }

        }

        sendAlertToCategoryTeam(alertItems);

        try {

            //generate products list that is to be uploaded in Amazon.

        	AmazonSCDataGenerator ascdGenerator = new AmazonSCDataGenerator(validEntities, GENERATION_TYPE);

            ascdGenerator.generateSCProdData();

        } catch (Exception e) {

        	e.printStackTrace();

            log.info("Could not generate Amazon data", e);

        }

    }

    private void sendAlertToCategoryTeam(List<Item> items) {

    	if(items!=null && items.size()!=0){

			GmailUtils util = new GmailUtils();

			String[] recipients = {"amit.gupta@shop2020.in", "chaitnaya.vats@shop2020.in", "ashutosh.saxena@shop2020.in"};

			String from = "build@shop2020.in";

			String password = "cafe@nes";

			String subject = Utils.EXPECTED_ARRIVAL_ACHIEVED_TEMPLATE;

			StringBuffer message = new StringBuffer("Please check the following items:\n");

			List<File> emptyList = new ArrayList<File>();

			for( Item item : items){

				message.append("\t" + getProductName(item));

			}

			try {

				util.sendSSLMessage(recipients, subject, message.toString(), from, password, emptyList);

			} catch (Exception e){

				log.info("Could not send alert" + e);

			}

    	}

	}

	private String getProductName(Item item) {

    	String brand = item.getBrand();

		String modelName = item.getModelName();

		String modelNumber = item.getModelNumber();

		String product = "";

		if(StringUtils.isEmpty(modelName)){

			product = brand + " " + modelNumber;

		}else {

			product = brand + " " + modelName + " " + modelNumber;

		}

		return product;

	}

    /**

     * Checks weather entity is valid or not. Entity will be invalid in one of

     * these cases:

     * <ol>

     * <li>The entity is not ready.

     * <li>The category has not been updated yet. (Set to -1).

     * <li>Content has not been updated after last content generation timestamp.

     * </ol>

     * 

     * @param entity

     * @return

     * @throws Exception

     */

    private boolean isValidEntity(Entity entity) throws Exception {

        ExpandedEntity expEntity = new ExpandedEntity(entity);

        EntityState state = CreationUtils.getEntityState(entity.getID());

        long categoryID = expEntity.getCategoryID();

        if (state.getStatus() != EntityStatus.READY || categoryID == -1) {

            return false;

        }

        if (state.getMerkedReadyOn().getTime() < this.lastGenerationTime) {

            return false;

        }

        return true;

    }

    private void populateEntityIdItemMap() {

        Date todate = new Date();

        Utils.info("Processing " + items.size() + " items");

        for (Item item : items) {

            Utils.info(item.getId() + ":" + item.getItemStatus() + ":" + item.getCatalogItemId());

            // TODO Can be removed as we are checking in calling function

            if (!(item.getItemStatus() == status.ACTIVE

                    || item.getItemStatus() == status.CONTENT_COMPLETE || item

                    .getItemStatus() == status.PAUSED || item.getItemStatus() == status.COMING_SOON)) {

                continue;

            }

            Utils.info(item.getStartDate() + ":" + item.getSellingPrice());

			if (todate.getTime() < item.getStartDate()

					&& (!item.isSetExpectedArrivalDate() || todate.getTime() < item.getComingSoonStartDate()) 

					|| item.getSellingPrice() == 0) {

				continue;

			}

            Utils.info(item.getId() + " Item is adding");

            List<Item> itemList = entityIdItemMap.get(item.getCatalogItemId());

            if (itemList == null) {

                itemList = new ArrayList<Item>();

            } 

            itemList.add(item);

            entityIdItemMap.put(item.getCatalogItemId(), itemList);

        }

        Utils.info("Processing " + entityIdItemMap.size() + " entities");

        // Remove all items which have not been updated since last content

        // generation.

        List<Long> removeEntities = new ArrayList<Long>();

        for (Long entityId : entityIdItemMap.keySet()) {

            boolean isValidEntity = false;

            // If any one of the items has been updated before current

            // timestamp, than we generate content for whole entity

            for (Item item : entityIdItemMap.get(entityId)) {

                if (item.getUpdatedOn() > lastGenerationTime || item.getItemStatus()==status.COMING_SOON) {

                    isValidEntity = true;

                }

            }

            if (!isValidEntity) {

                removeEntities.add(entityId);

            }

        }

        //Simply assign allValidEntityIds to a class variable as these need to be used where all valid entites

        //are needed.

        allValidEntityIds = new ArrayList<Long>(entityIdItemMap.keySet());

        for (Long entityId : removeEntities) {

            entityIdItemMap.remove(entityId);

        }

        Utils.info("Final valid entities to be processed: " + entityIdItemMap.size());

    }

    private void synonymTitlesExporter() {

    	SynonymExporter sx = new SynonymExporter();

        Map<Long, Map<String,List<String>>> synonyms = sx.getSynonyms();

        Map<String, List<String>> finalsynonyms = new HashMap<String, List<String>>();

    	for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {

            long entityId = entry.getKey();

	    	try{

	            String brand = "";

	            String originalModelName = "";

	            String originalModelNumber = "";

	            List<String> modelNameSynonyms =  new ArrayList<String>();

	            List<String> modelNumberSynonyms =  new ArrayList<String>();

	            List<String> titles = new ArrayList<String>();

	            Map<String,List<String>> synonymMap = synonyms.get(entityId);

	            if(synonymMap != null && !synonymMap.isEmpty()){

	            	if(synonymMap.get("ORIGINAL_MODEL_NAME") != null && !synonymMap.get("ORIGINAL_MODEL_NAME").isEmpty()){

	            		modelNameSynonyms.addAll(synonymMap.get("ORIGINAL_MODEL_NAME"));

	            		originalModelName = synonymMap.get("ORIGINAL_MODEL_NAME").get(0);

	            	}

	            	if(synonymMap.get("MODEL_NAME") != null && !synonymMap.get("MODEL_NAME").isEmpty()){

	            		modelNameSynonyms.addAll(synonymMap.get("MODEL_NAME"));

	            	}

	            	if(synonymMap.get("ORIGINAL_MODEL_NUMBER") != null && !synonymMap.get("ORIGINAL_MODEL_NUMBER").isEmpty()){

	            		modelNumberSynonyms.addAll(synonymMap.get("ORIGINAL_MODEL_NUMBER"));

	            		originalModelNumber = synonymMap.get("ORIGINAL_MODEL_NUMBER").get(0);

	            	}

	            	if(synonymMap.get("MODEL_NUMBER") != null && !synonymMap.get("MODEL_NUMBER").isEmpty()){

	            		modelNumberSynonyms.addAll(synonymMap.get("MODEL_NUMBER"));

	            	}

	            	brand = ((synonymMap.get("ORIGINAL_BRAND") != null && !synonymMap.get("ORIGINAL_BRAND").isEmpty()) ? synonymMap.get("ORIGINAL_BRAND").get(0) : "");

	            }

	            for(String model_name: modelNameSynonyms){

	            	for(String model_number: modelNumberSynonyms){

	            		String title = brand + " " + model_name + " " + model_number;

	            		title = title.replaceAll("  ", " ");

	            		titles.add(title);

	            	}

	            }

	            String originaltitle = brand + " " + originalModelName + " " + originalModelNumber;

	            originaltitle = originaltitle.replaceAll("  ", " ");

	            originaltitle = originaltitle.trim();

	            if(!originaltitle.isEmpty()) {

	            	finalsynonyms.put(originaltitle, titles);

	            }

	        } catch (Exception e) {

				e.printStackTrace();

			}

    	}

    	String autosuggestFilename = Utils.EXPORT_JAVASCRIPT_CONTENT_PATH + "autosuggest.json";

        Gson gson = new Gson();

		try {

			DBUtils.store(gson.toJson(finalsynonyms), autosuggestFilename);

		} catch (Exception e) {

			e.printStackTrace();

		}

    }

}