Subversion Repositories SmartDukaan

package in.shop2020.util;

import in.shop2020.metamodel.core.Entity;

import in.shop2020.metamodel.core.EntityState;

import in.shop2020.metamodel.core.EntityStatus;

import in.shop2020.metamodel.util.CreationUtils;

import in.shop2020.model.v1.catalog.CatalogService.Client;

import in.shop2020.model.v1.catalog.Item;

import in.shop2020.model.v1.catalog.Source;

import in.shop2020.model.v1.catalog.status;

import in.shop2020.thrift.clients.CatalogClient;

import in.shop2020.ui.util.ComparisonStatsFetcher;

import in.shop2020.ui.util.NewVUI;

import in.shop2020.ui.util.PriceInsertor;

import in.shop2020.ui.util.SpecialPageJSONConvertor;

import in.shop2020.utils.GmailUtils;

import java.io.File;

import java.io.FileNotFoundException;

import java.io.FileReader;

import java.io.IOException;

import java.io.Reader;

import java.text.SimpleDateFormat;

import java.util.ArrayList;

import java.util.Calendar;

import java.util.Date;

import java.util.HashMap;

import java.util.Iterator;

import java.util.LinkedHashMap;

import java.util.List;

import java.util.Map;

import javax.mail.MessagingException;

import org.apache.commons.cli.CommandLine;

import org.apache.commons.cli.CommandLineParser;

import org.apache.commons.cli.HelpFormatter;

import org.apache.commons.cli.Options;

import org.apache.commons.cli.ParseException;

import org.apache.commons.cli.PosixParser;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.google.gson.Gson;

import com.google.gson.reflect.TypeToken;

public class ContentGenerationUtility {

	private static final String UPDATE_TYPE_CATALOG = "CATALOG";

	private static final String UPDATE_TYPE_CONTENT = "CONTENT";

	private static final String COMMAND_LINE = "java ContentGenerationUtility.class -t { ALL | INCREMENTAL | ONE } -s { yyyy-MM-dd-HH-mm-ss } -u { CONTENT | CATALOG } -e {EntityId} ";

	private static Log log = LogFactory.getLog(ContentGenerationUtility.class);

	private static long ONE_DAY = 24 * 60 * 60 * 1000; // milliseconds in a day

	// Commandline options

	private static Options options = null;

	private static final String GENERATION_TYPE_INCREMENTAL = "INCREMENTAL";

	private static final String GENERATION_TYPE_ALL = "ALL";

	private static final String GENERATION_TYPE_ONE = "ONE";

	private static final String UPDATE_TYPE_OPTION = "u";

	private static final String GENERATION_TYPE_OPTION = "t";

	private static final String ENTITY_ID_OPTION = "e";

	private static final String TIMESTAMP_OPTION = "s";

	// Default values of cmdline options

	private static String UPDATE_TYPE = UPDATE_TYPE_CONTENT;

	private static String GENERATION_TYPE = GENERATION_TYPE_INCREMENTAL;

	private static String ENTITY_ID = "ALL";

	private static String[] DOMAINPATHS = Utils.DOMAIN_NAMES_FOR_CONTENT_GENERATION.split(";");

	private Date timeStamp = null;

	private CommandLine cmd = null;

	private Map<Long, List<Item>> entityIdItemMap = new LinkedHashMap<Long, List<Item>>();

	private Long lastGenerationTime = 0l;

	private Long lastGenerationTimeNonZero = CreationUtils.getLastContentGenerationTime();

	private Map<Long, Entity> entities;

	private List<Long> removeEntities = new ArrayList<Long>();

	private List<Item> items;

	private List<Source> sources;

	private CatalogClient csc;

	List<Entity> validPartialEntities = new ArrayList<Entity>();

	private Client client;

	private List<Item> alertItems;

	private long newLastGenerationTime;

	static {

		options = new Options();

		options.addOption(GENERATION_TYPE_OPTION, true, "Generation type");

		options.addOption(UPDATE_TYPE_OPTION, true, "Default is : " + UPDATE_TYPE);

		options.addOption(ENTITY_ID_OPTION, true, "all entities " + ENTITY_ID + " by default");

		options.addOption(TIMESTAMP_OPTION, true, "Manual timestamp");

	}

	public ContentGenerationUtility() throws Exception {

		csc = new CatalogClient();

		client = csc.getClient();

		sources = client.getAllSources();

		alertItems = new ArrayList<Item>();

	}

	/**

	 * @param args

	 * @throws Exception

	 */

	public static void main(String[] args) throws Exception {

		ContentGenerationUtility cgu = new ContentGenerationUtility();

		// Load arguments

		cgu.loadArgs(args);

		// Call method based on arguments

		cgu.callMethod();

	}

	/**

	 * Validate and set command line arguments. Exit after printing usage if

	 * anything is astray

	 * 

	 * @param args

	 *            String[] args as featured in public static void main()

	 */

	private void loadArgs(String[] args) {

		CommandLineParser parser = new PosixParser();

		try {

			cmd = parser.parse(options, args);

		} catch (ParseException e) {

			log.error("Error parsing arguments", e);

			System.exit(1);

		}

		// Check for mandatory args

		if (!(cmd.hasOption(GENERATION_TYPE_OPTION) && cmd.hasOption(UPDATE_TYPE_OPTION))) {

			HelpFormatter formatter = new HelpFormatter();

			formatter.printHelp(COMMAND_LINE, options);

			System.exit(1);

		}

		GENERATION_TYPE = cmd.getOptionValue(GENERATION_TYPE_OPTION);

		UPDATE_TYPE = cmd.getOptionValue(UPDATE_TYPE_OPTION);

		// Look for optional args.

		if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {

			if (cmd.hasOption(ENTITY_ID_OPTION)) {

				ENTITY_ID = cmd.getOptionValue(ENTITY_ID_OPTION);

			} else {

				HelpFormatter formatter = new HelpFormatter();

				formatter.printHelp(COMMAND_LINE, options);

				System.exit(1);

			}

		}

		if (GENERATION_TYPE_INCREMENTAL.equals(GENERATION_TYPE))

			if (cmd.hasOption(TIMESTAMP_OPTION)) {

				SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");

				try {

					timeStamp = df.parse(cmd.getOptionValue(TIMESTAMP_OPTION));

				} catch (Exception e) {

					HelpFormatter formatter = new HelpFormatter();

					formatter.printHelp(COMMAND_LINE, options);

					System.exit(1);

				}

			}

	}

	/**

	 * Call method based on arguments

	 * 

	 * @throws Exception

	 */

	private void callMethod() {

		boolean isSuccess = false;

		String logfile = "/tmp/content-from-cms.log";

		if (UPDATE_TYPE.equals(UPDATE_TYPE_CONTENT)) {

			logfile = "/tmp/content-from-cms.log";

			try {

				this.generateContent();

				isSuccess = true;

			} catch (Exception e) {

				log.error("Error generating content", e);

			}

		}

		if (UPDATE_TYPE.equals(UPDATE_TYPE_CATALOG)) {

			logfile = "/tmp/content-from-catalog.log";

			try {

				this.updatePrices();

				isSuccess = true;

			} catch (Exception e) {

				log.error("Error updating prices", e);

			}

		}

		GmailUtils gm = new GmailUtils();

		String[] sendTo = { "amit.gupta@shop2020.in" };

		try {

			gm.sendSSLMessage(sendTo, "Content Generation Successful ? : " + isSuccess,

					"Content generation completed at time : " + Calendar.getInstance().getTime().toString(),

					"build@shop2020.in", "cafe@nes", logfile);

		} catch (MessagingException e) {

			log.error("Could not send status mail", e);

		}

	}

	public boolean cleanDir(File dir, boolean deleteSelf) {

		if (dir.isDirectory()) {

			String[] children = dir.list();

			for (int i = 0; i < children.length; i++) {

				boolean success = cleanDir(new File(dir, children[i]), true);

				if (!success) {

					return false;

				}

			}

		}

		// The directory is now empty so delete it

		if (deleteSelf) {

			return dir.delete();

		}

		return true;

	}

	private void removeOldResources() throws IOException {

		File f = new File(Utils.EXPORT_SOLR_PATH);

		if (f.exists()) {

			cleanDir(f, false);

		}

		for (String domainPath : DOMAINPATHS) {

			String pathName = domainPath.split("\\.")[0].split(":")[0];

			File f1 = new File(Utils.EXPORT_PATH + "html/entities-" + pathName);

			if (f1.exists()) {

				cleanDir(f1, false);

			} else {

				f1.mkdir();

			}

		}

	}

	/**

	 * Update the prices in the generated content

	 * 

	 * @throws Exception

	 */

	private void updatePrices() throws Exception {

		lastGenerationTime = new Long(0);

		List<Long> activeItems = new ArrayList<Long>();

		if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {

			items = client.getItemsByCatalogId(Long.parseLong(ENTITY_ID));

			Iterator<Item> it = items.iterator();

			while (it.hasNext()) {

				Item ite = it.next();

				status st = ite.getItemStatus();

				if (!(st.equals(status.ACTIVE) || st.equals(status.PAUSED) || st.equals(status.COMING_SOON))) {

					it.remove();

				}

				if (st.equals(status.ACTIVE)) {

					activeItems.add(ite.getId());

				}

			}

			try {

				// Generate prices and availability data for amazon

				AmazonSCDataGenerator.generatePricesAndAvailability(items);

			} catch (Exception e) {

				log.info("Could not generate Amazon prices and availability", e);

			}

			// ProductListGenerator.updatePriceForEntity(Long.parseLong(ENTITY_ID),

			// items.get(0).getSellingPrice(), items.get(0).getMrp());

		} else {

			log.info("Before getting active items.");

			items = client.getAllItemsByStatus(status.ACTIVE);

			for (Item item : items) {

				activeItems.add(item.getId());

			}

			log.info("Before getting coming items.");

			items.addAll(client.getAllItemsByStatus(status.COMING_SOON));

			log.info("Before getting paused items.");

			items.addAll(client.getAllItemsByStatus(status.PAUSED));

			// Clean up the data from the solr directories.

			log.info("Before removing old resources.");

			removeOldResources();

		}

		/*

		 * if (GENERATION_TYPE.equals(GENERATION_TYPE_INCREMENTAL)) { }

		 */

		populateEntityIdItemMap();

		// Generate partners and json objects for phones only

		if (!GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {

			ProductListGenerator generator = new ProductListGenerator(entityIdItemMap);

			log.info("Before auto suggest json");

			synonymTitlesExporter();

			log.info("Before product list js.");

			generator.generateProductListJavascript();

		}

		PriceInsertor priceInserter = new PriceInsertor();

		Map<Long, List<String>> entityTags = client.getAllEntityTags();

		Map<Long, Integer> popularityMap = getPolularityMap();

		List<Long> pausedByRiskItems = getRiskyPausedItems();

		for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {

			long entityId = entry.getKey();

			List<Item> items = entry.getValue();

			double minPrice = 0d;

			// Evaluating availability

			String availability = "Out of Stock";

			for (Item i : items) {

				if (i.getItemStatus().equals(status.ACTIVE)) {

					if (!(i.isRisky() && pausedByRiskItems.contains(i.getId()))) {

						availability = "In Stock";

					}

				}

			}

			StringBuilder priceString = new StringBuilder();

			StringBuilder availabilityString = new StringBuilder();

			boolean domainOnce = true;

			boolean sourceOnce = true;

			for (String domainPath : DOMAINPATHS) {

				String domainName = domainPath;

				String pathName = domainPath.split("\\.")[0].split(":")[0];

				if (!removeEntities.contains(entityId)) {

					priceInserter.insertPriceInHtml(items, entityId, domainName,

							Utils.EXPORT_PATH + "html/entities-" + pathName + "/", null);

				}

				if (domainOnce) {

					minPrice = getMinPrice(items, entityId, null);

					priceString.append("<field name=\"F_50002\">" + minPrice + "</field>");

					availabilityString.append("<field name=\"F_50028\">" + availability + "</field>");

					if (entityTags.containsKey(entityId)) {

						List<String> tags = entityTags.get(entityId);

						for (String tag : tags) {

							availabilityString.append("\n<field name=\"F_50029\">" + tag + "</field>");

						}

					}

					if (popularityMap.containsKey(entityId)) {

						availabilityString

								.append("\n<field name=\"F_50030\">" + popularityMap.get(entityId) + "</field>");

					} else {

						availabilityString.append("\n<field name=\"F_50030\">" + "0" + "</field>");

					}

					domainOnce = false;

				}

				if (sources != null) {

					for (Source source : sources) {

						priceInserter.insertPriceInHtml(items, entityId, domainName,

								Utils.EXPORT_PATH + "html/entities-" + pathName + "/", source);

						if (sourceOnce) {

							minPrice = getMinPrice(items, entityId, source);

							priceString

									.append("<field name=\"F_50002_" + source.getId() + "\">" + minPrice + "</field>");

						}

					}

					sourceOnce = false;

				}

			}

			priceInserter.insertPriceInSolrData(entityId, priceString.toString(), availabilityString.toString());

		}

		priceInserter.copySolrSchemaFiles();

	}

	private Map<Long, Integer> getPolularityMap() {

		try {

			Reader reader = new FileReader(Utils.EXPORT_PATH + Utils.POPULARITY_JSON);

			Map<Long, Integer> result = new Gson().fromJson(reader, new TypeToken<Map<Long, Integer>>() {

			}.getType());

			if (result == null) {

				result = new HashMap<Long, Integer>();

			}

			return result;

		} catch (Exception e) {

			log.error("Could not read popularity file");

			e.printStackTrace();

			return new HashMap<Long, Integer>();

		}

	}

	private List<Long> getRiskyPausedItems() {

		try {

			Reader reader = new FileReader(Utils.EXPORT_PATH + Utils.RISKY_PAUSED_JSON);

			List<Long> result = new Gson().fromJson(reader, new TypeToken<List<Long>>() {

			}.getType());

			if (result == null) {

				result = new ArrayList<Long>();

			}

			return result;

		} catch (FileNotFoundException e) {

			log.error("Could not read paused file");

			e.printStackTrace();

			return new ArrayList<Long>();

		}

	}

	/**

     * Generates content for the specified entity embedding links to the

     * specified domain name.

     * 

     * The method will not generate content if one of the following conditions

     * is met:

     * <ol>

     * <li>The entity is not ready.

     * <li>The category has not been updated yet. (Set to -1).

     * <li>The content has not been updated.

     * </ol>

     * 

     * @throws

     */

    private void generateContent() throws Exception {

        if (GENERATION_TYPE.equals(GENERATION_TYPE_ALL)) {

            entities = CreationUtils.getEntities();

            lastGenerationTime = new Long(0);

            items = client.getAllItemsByStatus(status.CONTENT_COMPLETE);

            items.addAll(client.getAllItemsByStatus(status.COMING_SOON));

            //items.addAll(client.getAllItemsByStatus(status.PARTIALLY_ACTIVE));

            items.addAll(client.getAllItemsByStatus(status.ACTIVE));

            items.addAll(client.getAllItemsByStatus(status.PAUSED));

        } else if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {

            entities = new HashMap<Long, Entity>();

            entities.put(Long.parseLong(ENTITY_ID),

                    CreationUtils.getEntity(Long.parseLong(ENTITY_ID)));

            items = client.getItemsByCatalogId(Long.parseLong(ENTITY_ID));

            Iterator<Item> ite = items.iterator();

            while(ite.hasNext()) {

            	Item i = ite.next();

            	if(!(i.getItemStatus().equals(status.ACTIVE) || i.getItemStatus().equals(status.PAUSED) 

            	|| i.getItemStatus().equals(status.CONTENT_COMPLETE) || i.getItemStatus().equals(status.COMING_SOON))){

            		ite.remove();

            	}

            }

            lastGenerationTime = new Long(0);

        } else {

        	entities = CreationUtils.getEntities();

            //  When we read lastGenerationTime from database

            //  then only we should mark the 

            //	current time as newLastGenerationTime

        	newLastGenerationTime = new Date().getTime();

            if(timeStamp == null) {

            	lastGenerationTime = CreationUtils.getLastContentGenerationTime();

            } else {

            	lastGenerationTime = timeStamp.getTime();

            }

            log.info("lastGenerationTime: " + lastGenerationTime);

            if (lastGenerationTime == null) {

                lastGenerationTime = new Long(0);

            }

            items = client.getAllItemsByStatus(status.CONTENT_COMPLETE);

            items.addAll(client.getAllItemsByStatus(status.COMING_SOON));

            //items.addAll(client.getAllItemsByStatus(status.PARTIALLY_ACTIVE));

        } 

        // Filter invalid entities here

        List<Entity> validEntities = new ArrayList<Entity>();

        List<Long> validEntityIds = new ArrayList<Long>();

        for (long entityID : entities.keySet()) {

            if (isValidEntity(entities.get(entityID))) {

                validEntities.add(entities.get(entityID));

                validEntityIds.add(entityID);

            }

        }

        log.info("Generating synonyms");

        SynonymExporter sx = new SynonymExporter();

        sx.storeSynonyms(validEntities);

        // Calculate comparison scores

        log.info("Calculating comparison scores");

        NewCMP cmp = new NewCMP(validEntities);

        Map<Long, Map<Long, Double>> slideScoresByEntity = cmp.getSlideScores();

        CreationUtils.storeSlideScores(slideScoresByEntity);

        // Fetch comparison statistics everyday and store them in BDB

        //This might be remove. 

        log.info("Fetching comparison statistics");

        ComparisonStatsFetcher csf = new ComparisonStatsFetcher();

        csf.fetchAndStoreComparisonStats();

        populateEntityIdItemMap();

        log.info("Writing JSON file for special pages");

        SpecialPageJSONConvertor bjc = new SpecialPageJSONConvertor();

        bjc.writeToJSONFile(new File(Utils.EXPORT_JAVASCRIPT_CONTENT_PATH

                + "special-pages.json"));

        log.info("Generating velocity templates, images, documents etc.");

        NewVUI vui = new NewVUI(lastGenerationTime);

        for (Entity entity : validEntities) {

            log.info("Processing Entityid: " + entity.getID());

            vui.generateContentForOneEntity(entity, Utils.EXPORT_VELOCITY_PATH);

        }

        try {

        	/*AmazonSCDataGenerator ascdg = new AmazonSCDataGenerator(validPartialEntities);*/

        	AmazonSCDataGenerator ascdg = new AmazonSCDataGenerator(validPartialEntities,false);

        	ascdg.generateSCProdData();

        }catch (Exception e ){

        	log.info("Could not generate Jungle upload Feed");

        	e.printStackTrace();

        }

        if (newLastGenerationTime != 0) {

            CreationUtils.storeLastContentGenerationTime(newLastGenerationTime);

        }

        log.info("Generating Solr files");

        NewIR ir = new NewIR(validEntities);

        ir.exportIRData();

        // ir.transformIrDataXMLtoSolrXML();em.

        if(!GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {

	        ir.exportIRMetaData();

	        ir.transformIrMetaDataXMLtoSolrSchemaXML();

	        ir.transformIrMetaDataXMLtoSolrCatchAllXML();

        }

        csc = new CatalogClient();

        client = csc.getClient();

        for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {

            List<Item> items = entry.getValue();

            for (Item item : items) {

                if (item.getItemStatus() == status.CONTENT_COMPLETE 

                		|| item.getItemStatus() == status.COMING_SOON) {

                	if(item.getStartDate() <= new Date().getTime() + ONE_DAY){

                		item.setItemStatus(status.ACTIVE);

                		item.setStatus_description("This item is active");

                	} else {

                		item.setItemStatus(status.COMING_SOON);

                		String productName = getProductName(item);

                		String statusDescription = productName + " is coming soon.";

                		if(item.getExpectedArrivalDate()>new Date().getTime() + ONE_DAY){

                			statusDescription = productName + " will be available by " 

                			+ new SimpleDateFormat("dd/MM/yy").format(new Date(item.getExpectedArrivalDate()));

                		}

                		//Send alert to Category team one day before expected arrival date

                		//So they may change the expected arrival date if they want to.

                		if(item.getExpectedArrivalDate() < new Date().getTime() + 2*ONE_DAY && 

                				item.getExpectedArrivalDate() > new Date().getTime() + ONE_DAY) {

                				alertItems.add(item);

                		}

                		item.setStatus_description(statusDescription);

                	}

                	try {

                		client.updateItem(item);

                	}catch( Exception e){

                		csc = new CatalogClient();

                        client = csc.getClient();

                        log.info("Client generated again for item " + item.getId());

                        log.info("Skipped  item " + item.getId());

                        //client.updateItem(item);

                	}

            	}

            }

        }

        sendAlertToCategoryTeam(alertItems);

    }

	private void sendAlertToCategoryTeam(List<Item> items) {

		if (items != null && items.size() != 0) {

			GmailUtils util = new GmailUtils();

			String[] recipients = { "amit.gupta@shop2020.in", "chaitnaya.vats@shop2020.in",

					"khushal.bhatia@shop2020.in", "vrinda.k@shop2020.in" };

			String from = "build@shop2020.in";

			String password = "cafe@nes";

			String subject = Utils.EXPECTED_ARRIVAL_ACHIEVED_TEMPLATE;

			StringBuffer message = new StringBuffer("Please check the following items:\n");

			List<File> emptyList = new ArrayList<File>();

			for (Item item : items) {

				message.append("\t" + getProductName(item));

			}

			try {

				util.sendSSLMessage(recipients, subject, message.toString(), from, password, emptyList);

			} catch (Exception e) {

				log.info("Could not send alert" + e);

			}

		}

	}

	private String getProductName(Item item) {

		String brand = item.getBrand();

		String modelName = item.getModelName();

		String modelNumber = item.getModelNumber();

		String product = "";

		if (StringUtils.isEmpty(modelName)) {

			product = brand + " " + modelNumber;

		} else {

			product = brand + " " + modelName + " " + modelNumber;

		}

		return product;

	}

	/**

	 * Checks weather entity is valid or not. Entity will be invalid in one of these

	 * cases:

	 * <ol>

	 * <li>The entity is not ready.

	 * <li>The category has not been updated yet. (Set to -1).

	 * <li>Content has not been updated after last content generation timestamp.

	 * </ol>

	 * 

	 * @param entity

	 * @return

	 * @throws Exception

	 */

	private boolean isValidEntity(Entity entity) throws Exception {

		EntityState state = CreationUtils.getEntityState(entity.getID());

		long categoryID = state.getCategoryID();

		if (state.getStatus() != EntityStatus.READY || categoryID == -1) {

			return false;

		}

		if (state.getMerkedReadyOn().getTime() < this.lastGenerationTime) {

			return false;

		}

		if (state.getMerkedReadyOn().getTime() > this.lastGenerationTimeNonZero) {

			Utils.info("Added to Partail:" + entity.getID());

			validPartialEntities.add(entity);

		}

		return true;

	}

	private void populateEntityIdItemMap() {

		Date todate = new Date();

		Utils.info("Processing " + items.size() + " items");

		for (Item item : items) {

			Utils.info(item.getId() + ":" + item.getItemStatus() + ":" + item.getCatalogItemId());

			// TODO Can be removed as we are checking in calling function

			/*

			 * if (!(item.getItemStatus() == status.ACTIVE || item.getItemStatus() ==

			 * status.CONTENT_COMPLETE || item .getItemStatus() == status.PAUSED ||

			 * item.getItemStatus() == status.COMING_SOON)) { continue; }

			 */

			SimpleDateFormat df1 = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");

			Utils.info(df1.format(item.getStartDate()) + ":" + item.getSellingPrice());

			if (todate.getTime() < item.getStartDate()

					&& (!item.isSetExpectedArrivalDate() || todate.getTime() < item.getComingSoonStartDate())

					|| item.getSellingPrice() == 0) {

				continue;

			}

			Utils.info(item.getId() + " Item is adding");

			List<Item> itemList = entityIdItemMap.get(item.getCatalogItemId());

			if (itemList == null) {

				itemList = new ArrayList<Item>();

			}

			itemList.add(item);

			entityIdItemMap.put(item.getCatalogItemId(), itemList);

		}

		Utils.info("Processing " + entityIdItemMap.size() + " entities");

		// Remove all items which have not been updated since last content

		// generation.

		if (!(UPDATE_TYPE_CONTENT.equals(UPDATE_TYPE) || lastGenerationTime == 0)) {

			for (Long entityId : entityIdItemMap.keySet()) {

				boolean isValidEntity = false;

				// If any one of the items has been updated before current

				// timestamp, than we generate content for pricing

				for (Item item : entityIdItemMap.get(entityId)) {

					if (item.getUpdatedOn() > lastGenerationTime) {

						isValidEntity = true;

					}

				}

				if (!isValidEntity) {

					removeEntities.add(entityId);

				}

			}

		}

		Utils.info("Final valid entities to be processed: " + entityIdItemMap.size());

	}

	private void synonymTitlesExporter() {

		SynonymExporter sx = new SynonymExporter();

		Map<Long, Map<String, List<String>>> synonyms = sx.getSynonyms();

		Map<String, List<String>> finalsynonyms = new HashMap<String, List<String>>();

		for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {

			long entityId = entry.getKey();

			try {

				String brand = "";

				String originalModelName = "";

				String originalModelNumber = "";

				List<String> modelNameSynonyms = new ArrayList<String>();

				List<String> modelNumberSynonyms = new ArrayList<String>();

				List<String> titles = new ArrayList<String>();

				Map<String, List<String>> synonymMap = synonyms.get(entityId);

				if (synonymMap != null && !synonymMap.isEmpty()) {

					if (synonymMap.get("ORIGINAL_MODEL_NAME") != null

							&& !synonymMap.get("ORIGINAL_MODEL_NAME").isEmpty()) {

						modelNameSynonyms.addAll(synonymMap.get("ORIGINAL_MODEL_NAME"));

						originalModelName = synonymMap.get("ORIGINAL_MODEL_NAME").get(0);

					}

					if (synonymMap.get("MODEL_NAME") != null && !synonymMap.get("MODEL_NAME").isEmpty()) {

						modelNameSynonyms.addAll(synonymMap.get("MODEL_NAME"));

					}

					if (synonymMap.get("ORIGINAL_MODEL_NUMBER") != null

							&& !synonymMap.get("ORIGINAL_MODEL_NUMBER").isEmpty()) {

						modelNumberSynonyms.addAll(synonymMap.get("ORIGINAL_MODEL_NUMBER"));

						originalModelNumber = synonymMap.get("ORIGINAL_MODEL_NUMBER").get(0);

					}

					if (synonymMap.get("MODEL_NUMBER") != null && !synonymMap.get("MODEL_NUMBER").isEmpty()) {

						modelNumberSynonyms.addAll(synonymMap.get("MODEL_NUMBER"));

					}

					brand = ((synonymMap.get("ORIGINAL_BRAND") != null && !synonymMap.get("ORIGINAL_BRAND").isEmpty())

							? synonymMap.get("ORIGINAL_BRAND").get(0)

							: "");

				}

				for (String model_name : modelNameSynonyms) {

					for (String model_number : modelNumberSynonyms) {

						String title = brand + " " + model_name + " " + model_number;

						title = title.replaceAll("  ", " ");

						titles.add(title);

					}

				}

				String originaltitle = brand + " " + originalModelName + " " + originalModelNumber;

				originaltitle = originaltitle.replaceAll("  ", " ");

				originaltitle = originaltitle.trim();

				if (!originaltitle.isEmpty()) {

					finalsynonyms.put(originaltitle, titles);

				}

			} catch (Exception e) {

				e.printStackTrace();

			}

		}

		String autosuggestFilename = Utils.EXPORT_JAVASCRIPT_CONTENT_PATH + "autosuggest.json";

		Gson gson = new Gson();

		try {

			DBUtils.store(gson.toJson(finalsynonyms), autosuggestFilename);

		} catch (Exception e) {

			e.printStackTrace();

		}

	}

	public double getMinPrice(List<Item> items, long catalogId, Source source) {

		Item minPriceItem = null;

		for (Item item : items) {

			if (minPriceItem == null || minPriceItem.getSellingPrice() > item.getSellingPrice()) {

				minPriceItem = item;

			}

		}

		return minPriceItem.getSellingPrice();

	}

}