Subversion Repositories SmartDukaan

package in.shop2020.util;

import in.shop2020.metamodel.core.Entity;

import in.shop2020.metamodel.core.EntityState;

import in.shop2020.metamodel.core.EntityStatus;

import in.shop2020.metamodel.util.CreationUtils;

import in.shop2020.model.v1.catalog.CatalogService.Client;

import in.shop2020.model.v1.catalog.Item;

import in.shop2020.model.v1.catalog.Source;

import in.shop2020.model.v1.catalog.status;

import in.shop2020.thrift.clients.CatalogClient;

import in.shop2020.ui.util.ComparisonStatsFetcher;

import in.shop2020.ui.util.NewVUI;

import in.shop2020.ui.util.PriceInsertor;

import in.shop2020.ui.util.SpecialPageJSONConvertor;

import in.shop2020.utils.GmailUtils;

import java.io.File;

import java.io.FileNotFoundException;

import java.io.FileReader;

import java.io.IOException;

import java.io.Reader;

import java.text.SimpleDateFormat;

import java.util.ArrayList;

import java.util.Calendar;

import java.util.Date;

import java.util.HashMap;

import java.util.Iterator;

import java.util.LinkedHashMap;

import java.util.List;

import java.util.Map;

import javax.mail.MessagingException;

import org.apache.commons.cli.CommandLine;

import org.apache.commons.cli.CommandLineParser;

import org.apache.commons.cli.HelpFormatter;

import org.apache.commons.cli.Options;

import org.apache.commons.cli.ParseException;

import org.apache.commons.cli.PosixParser;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.google.gson.Gson;

import com.google.gson.reflect.TypeToken;

public class ContentGenerationUtility {

    private static final String   UPDATE_TYPE_CATALOG         = "CATALOG";

    private static final String   UPDATE_TYPE_CONTENT         = "CONTENT";

    private static final String   COMMAND_LINE                = "java ContentGenerationUtility.class -t { ALL | INCREMENTAL | ONE } -s { yyyy-MM-dd-HH-mm-ss } -u { CONTENT | CATALOG } -e {EntityId} ";

    private static Log            log                         = LogFactory

                                                                      .getLog(ContentGenerationUtility.class);

    private static long 			ONE_DAY					  = 24*60*60*1000;  	//milliseconds in a day						 	

    // Commandline options

    private static Options        options                     = null;

    private static final String   GENERATION_TYPE_INCREMENTAL = "INCREMENTAL";

    private static final String   GENERATION_TYPE_ALL         = "ALL";

    private static final String   GENERATION_TYPE_ONE         = "ONE";

    private static final String   UPDATE_TYPE_OPTION          = "u";

    private static final String   GENERATION_TYPE_OPTION      = "t";

    private static final String   ENTITY_ID_OPTION            = "e";

    private static final String   TIMESTAMP_OPTION            = "s";

    // Default values of cmdline options

    private static String         UPDATE_TYPE                 = UPDATE_TYPE_CONTENT;

    private static String         GENERATION_TYPE             = GENERATION_TYPE_INCREMENTAL;

    private static String         ENTITY_ID                   = "ALL";

    private static String [] 	  DOMAINPATHS 			      = Utils.DOMAIN_NAMES_FOR_CONTENT_GENERATION.split(";");

    private Date 				  timeStamp			  		  = null;

    private CommandLine           cmd                         = null;

    private Map<Long, List<Item>> entityIdItemMap             = new LinkedHashMap<Long, List<Item>>();

    private Long                  lastGenerationTime          = 0l;

    private Long                  lastGenerationTimeNonZero          = CreationUtils.getLastContentGenerationTime();

    private Map<Long, Entity>     entities;

    private List<Long> removeEntities = new ArrayList<Long>();

    private List<Item>            items;

    private List<Source>          sources;

    private CatalogClient         csc;

    List<Entity> validPartialEntities = new ArrayList<Entity>();

    private Client                client;

    private List<Item>			  alertItems;	

    private long                  newLastGenerationTime;

    static {

        options = new Options();

        options.addOption(GENERATION_TYPE_OPTION, true, "Generation type");

        options.addOption(UPDATE_TYPE_OPTION, true, "Default is : "

                + UPDATE_TYPE);

        options.addOption(ENTITY_ID_OPTION, true, "all entities " + ENTITY_ID

                + " by default");

        options.addOption(TIMESTAMP_OPTION, true, "Manual timestamp");

    }

    public ContentGenerationUtility() throws Exception {

        csc = new CatalogClient();

        client = csc.getClient();

        sources = client.getAllSources();

        alertItems = new ArrayList<Item>();	

    }

    /**

     * @param args

     * @throws Exception

     */

    public static void main(String[] args) throws Exception {

        ContentGenerationUtility cgu = new ContentGenerationUtility();

        // Load arguments

        cgu.loadArgs(args);

        // Call method based on arguments

        cgu.callMethod();

    }

    /**

     * Validate and set command line arguments. Exit after printing usage if

     * anything is astray

     * 

     * @param args

     *            String[] args as featured in public static void main()

     */

    private void loadArgs(String[] args) {

        CommandLineParser parser = new PosixParser();

        try {

            cmd = parser.parse(options, args);

        } catch (ParseException e) {

            log.error("Error parsing arguments", e);

            System.exit(1);

        }

        // Check for mandatory args

        if (!(cmd.hasOption(GENERATION_TYPE_OPTION) && cmd

                .hasOption(UPDATE_TYPE_OPTION))) {

            HelpFormatter formatter = new HelpFormatter();

            formatter.printHelp(COMMAND_LINE, options);

            System.exit(1);

        }

        GENERATION_TYPE = cmd.getOptionValue(GENERATION_TYPE_OPTION);

        UPDATE_TYPE = cmd.getOptionValue(UPDATE_TYPE_OPTION);

        // Look for optional args.

        if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {

            if (cmd.hasOption(ENTITY_ID_OPTION)) {

                ENTITY_ID = cmd.getOptionValue(ENTITY_ID_OPTION);

            } else {

                HelpFormatter formatter = new HelpFormatter();

                formatter.printHelp(COMMAND_LINE, options);

                System.exit(1);

            }

        }

        if (GENERATION_TYPE_INCREMENTAL.equals(GENERATION_TYPE))

        	if (cmd.hasOption(TIMESTAMP_OPTION)) {

        		SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");

        		try {

        		    timeStamp = df.parse(cmd.getOptionValue(TIMESTAMP_OPTION));

        		} catch(Exception e) {

        			HelpFormatter formatter = new HelpFormatter();

                    formatter.printHelp(COMMAND_LINE, options);

                    System.exit(1);

        		}

        	}

    }

    /**

     * Call method based on arguments

     * 

     * @throws Exception

     */

    private void callMethod() {

        boolean isSuccess = false;

        String logfile = "/tmp/content-from-cms.log";

        if (UPDATE_TYPE.equals(UPDATE_TYPE_CONTENT)) {

            logfile = "/tmp/content-from-cms.log";

            try {

                this.generateContent();

                isSuccess = true;

            } catch (Exception e) {

                log.error("Error generating content", e);

            }

        }

        if (UPDATE_TYPE.equals(UPDATE_TYPE_CATALOG)) {

            logfile = "/tmp/content-from-catalog.log";

            try {

                this.updatePrices();

                isSuccess = true;

            } catch (Exception e) {

                log.error("Error updating prices", e);

            }

        }

        GmailUtils gm = new GmailUtils();

        String[] sendTo = { "amit.gupta@shop2020.in" };

        try {

            gm.sendSSLMessage(sendTo, "Content Generation Successful ? : "

                    + isSuccess, "Content generation completed at time : "

                    + Calendar.getInstance().getTime().toString(),

                    "build@shop2020.in", "cafe@nes", logfile);

        } catch (MessagingException e) {

            log.error("Could not send status mail", e);

        }

    }

    public boolean cleanDir(File dir, boolean deleteSelf) {

        if (dir.isDirectory()) {

            String[] children = dir.list();

            for (int i = 0; i < children.length; i++) {

                boolean success = cleanDir(new File(dir, children[i]), true);

                if (!success) {

                    return false;

                }

            }

        }

        // The directory is now empty so delete it

        if (deleteSelf) {

            return dir.delete();

        }

        return true;

    }

    private void removeOldResources() throws IOException {

        File f = new File(Utils.EXPORT_SOLR_PATH);

        if (f.exists()) {

            cleanDir(f, false);

        }

        for(String domainPath : DOMAINPATHS){

        	String pathName = domainPath.split("\\.")[0].split(":")[0];

        	File f1 = new File(Utils.EXPORT_PATH + "html/entities-" +  pathName);

        	if (f1.exists()) {

        		cleanDir(f1, false);

        	}else {

        		f1.mkdir();

        	}

        }

    }

    /**

     * Update the prices in the generated content

     * 

     * @throws Exception

     */

    private void updatePrices() throws Exception {

        lastGenerationTime = new Long(0);

        List<Long> activeItems = new ArrayList<Long>();

        if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {

            items = client.getItemsByCatalogId(Long.parseLong(ENTITY_ID));

            Iterator<Item> it = items.iterator();

            while(it.hasNext()){

            	Item ite = it.next();

            	status st = ite.getItemStatus();

            	if(!(st.equals(status.ACTIVE) || st.equals(status.PAUSED) || 

            				st.equals(status.COMING_SOON))){

            		it.remove();

            	}

            	if(st.equals(status.ACTIVE)){

            		activeItems.add(ite.getId());

            	}

            }

            try {

            	//Generate prices and availability data for amazon

            	AmazonSCDataGenerator.generatePricesAndAvailability(items);

            } catch (Exception e) {

            	log.info("Could not generate Amazon prices and availability", e);

            }

            //ProductListGenerator.updatePriceForEntity(Long.parseLong(ENTITY_ID), items.get(0).getSellingPrice(), items.get(0).getMrp());

        } else {

        	log.info("Before getting active items.");

            items = client.getAllItemsByStatus(status.ACTIVE);

            for (Item item : items) {

            	activeItems.add(item.getId());

            }

            log.info("Before getting coming items.");

            items.addAll(client.getAllItemsByStatus(status.COMING_SOON));

            log.info("Before getting paused items.");

            items.addAll(client.getAllItemsByStatus(status.PAUSED));

            // Clean up the data from the solr directories.

            log.info("Before removing old resources.");

            removeOldResources();

        }

/*        if (GENERATION_TYPE.equals(GENERATION_TYPE_INCREMENTAL)) {

        }

*/

        populateEntityIdItemMap();

        // Generate partners and json objects for phones only

        if (!GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {

        	ProductListGenerator generator = new ProductListGenerator(entityIdItemMap);

        	log.info("Before auto suggest json");

        	synonymTitlesExporter();

        	log.info("Before product list js.");

        	generator.generateProductListJavascript(); 

        }

        PriceInsertor priceInserter = new PriceInsertor();

        Map<Long,List<String>> entityTags = client.getAllEntityTags();

        Map<Long, Integer> popularityMap = getPolularityMap();

        List<Long> pausedByRiskItems = getRiskyPausedItems();

        for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {

            long entityId = entry.getKey();

            List<Item> items = entry.getValue();

            double minPrice = 0d;

            //Evaluating availability

            String availability = "Out of Stock";

            for(Item i: items){

            	if(i.getItemStatus().equals(status.ACTIVE)) {

            		if(!(i.isRisky() && pausedByRiskItems.contains(i.getId()) )){

            			availability = "In Stock";

            		}

            	}

            }

            StringBuilder priceString = new StringBuilder();

            StringBuilder availabilityString = new StringBuilder();

            boolean domainOnce = true;

            boolean sourceOnce = true;

            for(String domainPath : DOMAINPATHS){

            	String domainName = domainPath;

            	String pathName = domainPath.split("\\.")[0].split(":")[0];

            	if(!removeEntities.contains(entityId)){

            		priceInserter.insertPriceInHtml(items, entityId, domainName, Utils.EXPORT_PATH + "html/entities-" +  pathName + "/", null);

            	}

            	if(domainOnce){

            		minPrice = getMinPrice(items, entityId, null);

            		priceString.append("<field name=\"F_50002\">" + minPrice + "</field>");

            		availabilityString.append("<field name=\"F_50028\">" + availability + "</field>");

            		if(entityTags.containsKey(entityId)) {

            			List<String> tags = entityTags.get(entityId);

            			for(String tag: tags){

            				availabilityString.append("\n<field name=\"F_50029\">" + tag + "</field>");

            			}

            		}

            		if(popularityMap.containsKey(entityId)) {

            			availabilityString.append("\n<field name=\"F_50030\">" + popularityMap.get(entityId) + "</field>");

            		}else {

            			availabilityString.append("\n<field name=\"F_50030\">" + "0" + "</field>");

            		}

            		domainOnce = false;

            	}

            	if(sources != null){

            		for (Source source : sources) {

						priceInserter.insertPriceInHtml(items, entityId,domainName, Utils.EXPORT_PATH + "html/entities-" + pathName + "/",source);

                        if(sourceOnce){

                        	minPrice = getMinPrice(items, entityId, source);

                        	priceString.append("<field name=\"F_50002_"

                                + source.getId() + "\">" + minPrice + "</field>");

                        }

            		}

            		sourceOnce = false;

            	}

            }

            priceInserter.insertPriceInSolrData(entityId,

                    priceString.toString(), availabilityString.toString());

        }

        priceInserter.copySolrSchemaFiles();

    }

	private Map<Long, Integer> getPolularityMap() {

		try {

			Reader reader = new FileReader(Utils.EXPORT_PATH + Utils.POPULARITY_JSON);

			return new Gson().fromJson(reader, new TypeToken<Map<Long, Integer>>() {}.getType());

		} catch (Exception e) {

			log.error("Could not read popularity file");

			e.printStackTrace();

			return new HashMap<Long, Integer>();

		}

	}

	private List<Long> getRiskyPausedItems() {

		try {

			Reader reader = new FileReader(Utils.EXPORT_PATH + Utils.RISKY_PAUSED_JSON);

			return new Gson().fromJson(reader, new TypeToken<List<Long>>() {}.getType());

		} catch (FileNotFoundException e) {

			log.error("Could not read paused file");

			e.printStackTrace();

			return new ArrayList<Long>();

		}

	}

	/**

     * Generates content for the specified entity embedding links to the

     * specified domain name.

     * 

     * The method will not generate content if one of the following conditions

     * is met:

     * <ol>

     * <li>The entity is not ready.

     * <li>The category has not been updated yet. (Set to -1).

     * <li>The content has not been updated.

     * </ol>

     * 

     * @throws

     */

    private void generateContent() throws Exception {

        if (GENERATION_TYPE.equals(GENERATION_TYPE_ALL)) {

            entities = CreationUtils.getEntities();

            lastGenerationTime = new Long(0);

            items = client.getAllItemsByStatus(status.CONTENT_COMPLETE);

            items.addAll(client.getAllItemsByStatus(status.COMING_SOON));

            //items.addAll(client.getAllItemsByStatus(status.PARTIALLY_ACTIVE));

            items.addAll(client.getAllItemsByStatus(status.ACTIVE));

            items.addAll(client.getAllItemsByStatus(status.PAUSED));

        } else if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {

            entities = new HashMap<Long, Entity>();

            entities.put(Long.parseLong(ENTITY_ID),

                    CreationUtils.getEntity(Long.parseLong(ENTITY_ID)));

            items = client.getItemsByCatalogId(Long.parseLong(ENTITY_ID));

            Iterator<Item> ite = items.iterator();

            while(ite.hasNext()) {

            	Item i = ite.next();

            	if(!(i.getItemStatus().equals(status.ACTIVE) || i.getItemStatus().equals(status.PAUSED) 

            	|| i.getItemStatus().equals(status.CONTENT_COMPLETE) || i.getItemStatus().equals(status.COMING_SOON))){

            		ite.remove();

            	}

            }

            lastGenerationTime = new Long(0);

        } else {

        	entities = CreationUtils.getEntities();

            //  When we read lastGenerationTime from database

            //  then only we should mark the 

            //	current time as newLastGenerationTime

        	newLastGenerationTime = new Date().getTime();

            if(timeStamp == null) {

            	lastGenerationTime = CreationUtils.getLastContentGenerationTime();

            } else {

            	lastGenerationTime = timeStamp.getTime();

            }

            log.info("lastGenerationTime: " + lastGenerationTime);

            if (lastGenerationTime == null) {

                lastGenerationTime = new Long(0);

            }

            items = client.getAllItemsByStatus(status.CONTENT_COMPLETE);

            items.addAll(client.getAllItemsByStatus(status.COMING_SOON));

            //items.addAll(client.getAllItemsByStatus(status.PARTIALLY_ACTIVE));

        } 

        // Filter invalid entities here

        List<Entity> validEntities = new ArrayList<Entity>();

        List<Long> validEntityIds = new ArrayList<Long>();

        for (long entityID : entities.keySet()) {

            if (isValidEntity(entities.get(entityID))) {

                validEntities.add(entities.get(entityID));

                validEntityIds.add(entityID);

            }

        }

        log.info("Generating synonyms");

        SynonymExporter sx = new SynonymExporter();

        sx.storeSynonyms(validEntities);

        // Calculate comparison scores

        log.info("Calculating comparison scores");

        NewCMP cmp = new NewCMP(validEntities);

        Map<Long, Map<Long, Double>> slideScoresByEntity = cmp.getSlideScores();

        CreationUtils.storeSlideScores(slideScoresByEntity);

        // Fetch comparison statistics everyday and store them in BDB

        //This might be remove. 

        log.info("Fetching comparison statistics");

        ComparisonStatsFetcher csf = new ComparisonStatsFetcher();

        csf.fetchAndStoreComparisonStats();

        populateEntityIdItemMap();

        log.info("Writing JSON file for special pages");

        SpecialPageJSONConvertor bjc = new SpecialPageJSONConvertor();

        bjc.writeToJSONFile(new File(Utils.EXPORT_JAVASCRIPT_CONTENT_PATH

                + "special-pages.json"));

        log.info("Generating velocity templates, images, documents etc.");

        NewVUI vui = new NewVUI(lastGenerationTime);

        for (Entity entity : validEntities) {

            log.info("Processing Entityid: " + entity.getID());

            vui.generateContentForOneEntity(entity, Utils.EXPORT_VELOCITY_PATH);

        }

        try {

        	AmazonSCDataGenerator ascdg = new AmazonSCDataGenerator(validPartialEntities);

        	ascdg.generateSCProdData();

        }catch (Exception e ){

        	log.info("Could not generate Jungle upload Feed");

        	e.printStackTrace();

        }

        if (newLastGenerationTime != 0) {

            CreationUtils.storeLastContentGenerationTime(newLastGenerationTime);

        }

        log.info("Generating Solr files");

        NewIR ir = new NewIR(validEntities);

        ir.exportIRData();

        // ir.transformIrDataXMLtoSolrXML();

        if(!GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {

	        ir.exportIRMetaData();

	        ir.transformIrMetaDataXMLtoSolrSchemaXML();

	        ir.transformIrMetaDataXMLtoSolrCatchAllXML();

        }

        for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {

            List<Item> items = entry.getValue();

            for (Item item : items) {

                if (item.getItemStatus() == status.CONTENT_COMPLETE 

                		|| item.getItemStatus() == status.COMING_SOON) {

                	if(item.getStartDate() <= new Date().getTime() + ONE_DAY){

                		item.setItemStatus(status.ACTIVE);

                		item.setStatus_description("This item is active");

                	} else {

                		item.setItemStatus(status.COMING_SOON);

                		String productName = getProductName(item);

                		String statusDescription = productName + " is coming soon.";

                		if(item.getExpectedArrivalDate()>new Date().getTime() + ONE_DAY){

                			statusDescription = productName + " will be available by " 

                			+ new SimpleDateFormat("dd/MM/yy").format(new Date(item.getExpectedArrivalDate()));

                		}

                		//Send alert to Category team one day before expected arrival date

                		//So they may change the expected arrival date if they want to.

                		if(item.getExpectedArrivalDate() < new Date().getTime() + 2*ONE_DAY && 

                				item.getExpectedArrivalDate() > new Date().getTime() + ONE_DAY) {

                				alertItems.add(item);

                		}

                		item.setStatus_description(statusDescription);

                	}

                    client.updateItem(item);

            	}

            }

        }

        sendAlertToCategoryTeam(alertItems);

    }

    private void sendAlertToCategoryTeam(List<Item> items) {

    	if(items!=null && items.size()!=0){

			GmailUtils util = new GmailUtils();

			String[] recipients = {"amit.gupta@shop2020.in", "chaitnaya.vats@shop2020.in", "khushal.bhatia@shop2020.in", "vrinda.k@shop2020.in"};

			String from = "build@shop2020.in";

			String password = "cafe@nes";

			String subject = Utils.EXPECTED_ARRIVAL_ACHIEVED_TEMPLATE;

			StringBuffer message = new StringBuffer("Please check the following items:\n");

			List<File> emptyList = new ArrayList<File>();

			for( Item item : items){

				message.append("\t" + getProductName(item));

			}

			try {

				util.sendSSLMessage(recipients, subject, message.toString(), from, password, emptyList);

			} catch (Exception e){

				log.info("Could not send alert" + e);

			}

    	}

	}

	private String getProductName(Item item) {

    	String brand = item.getBrand();

		String modelName = item.getModelName();

		String modelNumber = item.getModelNumber();

		String product = "";

		if(StringUtils.isEmpty(modelName)){

			product = brand + " " + modelNumber;

		}else {

			product = brand + " " + modelName + " " + modelNumber;

		}

		return product;

	}

    /**

     * Checks weather entity is valid or not. Entity will be invalid in one of

     * these cases:

     * <ol>

     * <li>The entity is not ready.

     * <li>The category has not been updated yet. (Set to -1).

     * <li>Content has not been updated after last content generation timestamp.

     * </ol>

     * 

     * @param entity

     * @return

     * @throws Exception

     */

    private boolean isValidEntity(Entity entity) throws Exception {

        EntityState state = CreationUtils.getEntityState(entity.getID());

        long categoryID = state.getCategoryID();

        if (state.getStatus() != EntityStatus.READY || categoryID == -1) {

            return false;

        }

        if(state.getMerkedReadyOn().getTime() < this.lastGenerationTime) {

            return false;

        }

    	if(state.getMerkedReadyOn().getTime() > this.lastGenerationTimeNonZero){

        		Utils.info("Added to Partail:" + entity.getID());

        		validPartialEntities.add(entity);

        }

        return true;

    }

    private void populateEntityIdItemMap() {

        Date todate = new Date();

        Utils.info("Processing " + items.size() + " items");

        for (Item item : items) {

            Utils.info(item.getId() + ":" + item.getItemStatus() + ":" + item.getCatalogItemId());

            // TODO Can be removed as we are checking in calling function

            /*if (!(item.getItemStatus() == status.ACTIVE

                    || item.getItemStatus() == status.CONTENT_COMPLETE || item

                    .getItemStatus() == status.PAUSED || item.getItemStatus() == status.COMING_SOON)) {

                continue;

            }*/

            SimpleDateFormat df1 = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");

            Utils.info(df1.format(item.getStartDate()) + ":" + item.getSellingPrice());

			if (todate.getTime() < item.getStartDate()

					&& (!item.isSetExpectedArrivalDate() || todate.getTime() < item.getComingSoonStartDate()) 

					|| item.getSellingPrice() == 0) {

				continue;

			}

            Utils.info(item.getId() + " Item is adding");

            List<Item> itemList = entityIdItemMap.get(item.getCatalogItemId());

            if (itemList == null) {

                itemList = new ArrayList<Item>();

            } 

            itemList.add(item);

            entityIdItemMap.put(item.getCatalogItemId(), itemList);

        }

        Utils.info("Processing " + entityIdItemMap.size() + " entities");

        // Remove all items which have not been updated since last content

        // generation.

        if (!(UPDATE_TYPE_CONTENT.equals(UPDATE_TYPE) || lastGenerationTime == 0)) {

	        for (Long entityId : entityIdItemMap.keySet()) {

	            boolean isValidEntity = false;

	            // If any one of the items has been updated before current

	            // timestamp, than we generate content for pricing

	            for (Item item : entityIdItemMap.get(entityId)) {

	                if (item.getUpdatedOn() > lastGenerationTime) {

	                    isValidEntity = true;

	                }

	            }

	            if (!isValidEntity) {

	                removeEntities.add(entityId);

	            }

	        }

        }

        Utils.info("Final valid entities to be processed: " + entityIdItemMap.size());

    }

    private void synonymTitlesExporter() {

    	SynonymExporter sx = new SynonymExporter();

        Map<Long, Map<String,List<String>>> synonyms = sx.getSynonyms();

        Map<String, List<String>> finalsynonyms = new HashMap<String, List<String>>();

    	for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {

            long entityId = entry.getKey();

	    	try{

	            String brand = "";

	            String originalModelName = "";

	            String originalModelNumber = "";

	            List<String> modelNameSynonyms =  new ArrayList<String>();

	            List<String> modelNumberSynonyms =  new ArrayList<String>();

	            List<String> titles = new ArrayList<String>();

	            Map<String,List<String>> synonymMap = synonyms.get(entityId);

	            if(synonymMap != null && !synonymMap.isEmpty()){

	            	if(synonymMap.get("ORIGINAL_MODEL_NAME") != null && !synonymMap.get("ORIGINAL_MODEL_NAME").isEmpty()){

	            		modelNameSynonyms.addAll(synonymMap.get("ORIGINAL_MODEL_NAME"));

	            		originalModelName = synonymMap.get("ORIGINAL_MODEL_NAME").get(0);

	            	}

	            	if(synonymMap.get("MODEL_NAME") != null && !synonymMap.get("MODEL_NAME").isEmpty()){

	            		modelNameSynonyms.addAll(synonymMap.get("MODEL_NAME"));

	            	}

	            	if(synonymMap.get("ORIGINAL_MODEL_NUMBER") != null && !synonymMap.get("ORIGINAL_MODEL_NUMBER").isEmpty()){

	            		modelNumberSynonyms.addAll(synonymMap.get("ORIGINAL_MODEL_NUMBER"));

	            		originalModelNumber = synonymMap.get("ORIGINAL_MODEL_NUMBER").get(0);

	            	}

	            	if(synonymMap.get("MODEL_NUMBER") != null && !synonymMap.get("MODEL_NUMBER").isEmpty()){

	            		modelNumberSynonyms.addAll(synonymMap.get("MODEL_NUMBER"));

	            	}

	            	brand = ((synonymMap.get("ORIGINAL_BRAND") != null && !synonymMap.get("ORIGINAL_BRAND").isEmpty()) ? synonymMap.get("ORIGINAL_BRAND").get(0) : "");

	            }

	            for(String model_name: modelNameSynonyms){

	            	for(String model_number: modelNumberSynonyms){

	            		String title = brand + " " + model_name + " " + model_number;

	            		title = title.replaceAll("  ", " ");

	            		titles.add(title);

	            	}

	            }

	            String originaltitle = brand + " " + originalModelName + " " + originalModelNumber;

	            originaltitle = originaltitle.replaceAll("  ", " ");

	            originaltitle = originaltitle.trim();

	            if(!originaltitle.isEmpty()) {

	            	finalsynonyms.put(originaltitle, titles);

	            }

	        } catch (Exception e) {

				e.printStackTrace();

			}

    	}

    	String autosuggestFilename = Utils.EXPORT_JAVASCRIPT_CONTENT_PATH + "autosuggest.json";

        Gson gson = new Gson();

		try {

			DBUtils.store(gson.toJson(finalsynonyms), autosuggestFilename);

		} catch (Exception e) {

			e.printStackTrace();

		}

    }

	public double getMinPrice(List<Item> items, long catalogId, Source source) {

		Item minPriceItem = null;

		for (Item item : items) {

			if (minPriceItem == null

					|| minPriceItem.getSellingPrice() > item.getSellingPrice()) {

				minPriceItem = item;

			}

		}

		return minPriceItem.getSellingPrice();

	}

}