Subversion Repositories SmartDukaan

package in.shop2020.util;

import in.shop2020.metamodel.core.Entity;
import in.shop2020.metamodel.core.EntityState;
import in.shop2020.metamodel.core.EntityStatus;
import in.shop2020.metamodel.util.CreationUtils;
import in.shop2020.model.v1.catalog.CatalogService.Client;
import in.shop2020.model.v1.catalog.Item;
import in.shop2020.model.v1.catalog.Source;
import in.shop2020.model.v1.catalog.status;
import in.shop2020.thrift.clients.CatalogClient;
import in.shop2020.ui.util.ComparisonStatsFetcher;
import in.shop2020.ui.util.NewVUI;
import in.shop2020.ui.util.PriceInsertor;
import in.shop2020.ui.util.SpecialPageJSONConvertor;
import in.shop2020.utils.GmailUtils;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

import javax.mail.MessagingException;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;

public class ContentGenerationUtility {
        private static final String UPDATE_TYPE_CATALOG = "CATALOG";

        private static final String UPDATE_TYPE_CONTENT = "CONTENT";

        private static final String COMMAND_LINE = "java ContentGenerationUtility.class -t { ALL | INCREMENTAL | ONE } -s { yyyy-MM-dd-HH-mm-ss } -u { CONTENT | CATALOG } -e {EntityId} ";

        private static Log log = LogFactory.getLog(ContentGenerationUtility.class);
        private static long ONE_DAY = 24 * 60 * 60 * 1000; // milliseconds in a day
        // Commandline options
        private static Options options = null;
        private static final String GENERATION_TYPE_INCREMENTAL = "INCREMENTAL";
        private static final String GENERATION_TYPE_ALL = "ALL";
        private static final String GENERATION_TYPE_ONE = "ONE";
        private static final String UPDATE_TYPE_OPTION = "u";
        private static final String GENERATION_TYPE_OPTION = "t";
        private static final String ENTITY_ID_OPTION = "e";
        private static final String TIMESTAMP_OPTION = "s";

        // Default values of cmdline options
        private static String UPDATE_TYPE = UPDATE_TYPE_CONTENT;
        private static String GENERATION_TYPE = GENERATION_TYPE_INCREMENTAL;
        private static String ENTITY_ID = "ALL";
        private static String[] DOMAINPATHS = Utils.DOMAIN_NAMES_FOR_CONTENT_GENERATION.split(";");

        private Date timeStamp = null;
        private CommandLine cmd = null;
        private Map<Long, List<Item>> entityIdItemMap = new LinkedHashMap<Long, List<Item>>();
        private Long lastGenerationTime = 0l;
        private Long lastGenerationTimeNonZero = CreationUtils.getLastContentGenerationTime();
        private Map<Long, Entity> entities;
        private List<Long> removeEntities = new ArrayList<Long>();
        private List<Item> items;
        private List<Source> sources;
        private CatalogClient csc;
        List<Entity> validPartialEntities = new ArrayList<Entity>();
        private Client client;
        private List<Item> alertItems;
        private long newLastGenerationTime;

        static {
                options = new Options();
                options.addOption(GENERATION_TYPE_OPTION, true, "Generation type");
                options.addOption(UPDATE_TYPE_OPTION, true, "Default is : " + UPDATE_TYPE);
                options.addOption(ENTITY_ID_OPTION, true, "all entities " + ENTITY_ID + " by default");
                options.addOption(TIMESTAMP_OPTION, true, "Manual timestamp");

        }

        public ContentGenerationUtility() throws Exception {
                csc = new CatalogClient();
                client = csc.getClient();
                sources = client.getAllSources();
                alertItems = new ArrayList<Item>();
        }

        /**
         * @param args
         * @throws Exception
         */
        public static void main(String[] args) throws Exception {
                ContentGenerationUtility cgu = new ContentGenerationUtility();

                // Load arguments
                cgu.loadArgs(args);

                // Call method based on arguments
                cgu.callMethod();
        }

        /**
         * Validate and set command line arguments. Exit after printing usage if
         * anything is astray
         * 
         * @param args
         *            String[] args as featured in public static void main()
         */
        private void loadArgs(String[] args) {
                CommandLineParser parser = new PosixParser();

                try {
                        cmd = parser.parse(options, args);
                } catch (ParseException e) {
                        log.error("Error parsing arguments", e);
                        System.exit(1);
                }

                // Check for mandatory args
                if (!(cmd.hasOption(GENERATION_TYPE_OPTION) && cmd.hasOption(UPDATE_TYPE_OPTION))) {
                        HelpFormatter formatter = new HelpFormatter();
                        formatter.printHelp(COMMAND_LINE, options);
                        System.exit(1);
                }

                GENERATION_TYPE = cmd.getOptionValue(GENERATION_TYPE_OPTION);

                UPDATE_TYPE = cmd.getOptionValue(UPDATE_TYPE_OPTION);

                // Look for optional args.
                if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {
                        if (cmd.hasOption(ENTITY_ID_OPTION)) {
                                ENTITY_ID = cmd.getOptionValue(ENTITY_ID_OPTION);
                        } else {
                                HelpFormatter formatter = new HelpFormatter();
                                formatter.printHelp(COMMAND_LINE, options);
                                System.exit(1);
                        }
                }

                if (GENERATION_TYPE_INCREMENTAL.equals(GENERATION_TYPE))
                        if (cmd.hasOption(TIMESTAMP_OPTION)) {
                                SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
                                try {
                                        timeStamp = df.parse(cmd.getOptionValue(TIMESTAMP_OPTION));
                                } catch (Exception e) {
                                        HelpFormatter formatter = new HelpFormatter();
                                        formatter.printHelp(COMMAND_LINE, options);
                                        System.exit(1);
                                }
                        }
        }

        /**
         * Call method based on arguments
         * 
         * @throws Exception
         */
        private void callMethod() {
                boolean isSuccess = false;
                String logfile = "/tmp/content-from-cms.log";
                if (UPDATE_TYPE.equals(UPDATE_TYPE_CONTENT)) {
                        logfile = "/tmp/content-from-cms.log";
                        try {
                                this.generateContent();
                                isSuccess = true;
                        } catch (Exception e) {
                                log.error("Error generating content", e);
                        }
                }

                if (UPDATE_TYPE.equals(UPDATE_TYPE_CATALOG)) {
                        logfile = "/tmp/content-from-catalog.log";
                        try {
                                this.updatePrices();
                                isSuccess = true;
                        } catch (Exception e) {
                                log.error("Error updating prices", e);
                        }
                }

                GmailUtils gm = new GmailUtils();
                String[] sendTo = { "amit.gupta@shop2020.in" };

                try {
                        gm.sendSSLMessage(sendTo, "Content Generation Successful ? : " + isSuccess,
                                        "Content generation completed at time : " + Calendar.getInstance().getTime().toString(),
                                        "build@shop2020.in", "cafe@nes", logfile);
                } catch (MessagingException e) {
                        log.error("Could not send status mail", e);
                }
        }

        public boolean cleanDir(File dir, boolean deleteSelf) {
                if (dir.isDirectory()) {
                        String[] children = dir.list();
                        for (int i = 0; i < children.length; i++) {
                                boolean success = cleanDir(new File(dir, children[i]), true);
                                if (!success) {
                                        return false;
                                }
                        }
                }

                // The directory is now empty so delete it
                if (deleteSelf) {
                        return dir.delete();
                }

                return true;
        }

        private void removeOldResources() throws IOException {
                File f = new File(Utils.EXPORT_SOLR_PATH);
                if (f.exists()) {
                        cleanDir(f, false);
                }

                for (String domainPath : DOMAINPATHS) {
                        String pathName = domainPath.split("\\.")[0].split(":")[0];
                        File f1 = new File(Utils.EXPORT_PATH + "html/entities-" + pathName);
                        if (f1.exists()) {
                                cleanDir(f1, false);
                        } else {
                                f1.mkdir();
                        }
                }
        }

        /**
         * Update the prices in the generated content
         * 
         * @throws Exception
         */
        private void updatePrices() throws Exception {
                lastGenerationTime = new Long(0);
                List<Long> activeItems = new ArrayList<Long>();
                if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {
                        items = client.getItemsByCatalogId(Long.parseLong(ENTITY_ID));
                        Iterator<Item> it = items.iterator();
                        while (it.hasNext()) {
                                Item ite = it.next();
                                status st = ite.getItemStatus();
                                if (!(st.equals(status.ACTIVE) || st.equals(status.PAUSED) || st.equals(status.COMING_SOON))) {
                                        it.remove();
                                }
                                if (st.equals(status.ACTIVE)) {
                                        activeItems.add(ite.getId());
                                }
                        }
                        try {
                                // Generate prices and availability data for amazon
                                AmazonSCDataGenerator.generatePricesAndAvailability(items);
                        } catch (Exception e) {
                                log.info("Could not generate Amazon prices and availability", e);
                        }
                        // ProductListGenerator.updatePriceForEntity(Long.parseLong(ENTITY_ID),
                        // items.get(0).getSellingPrice(), items.get(0).getMrp());
                } else {
                        log.info("Before getting active items.");
                        items = client.getAllItemsByStatus(status.ACTIVE);
                        for (Item item : items) {
                                activeItems.add(item.getId());
                        }
                        log.info("Before getting coming items.");
                        items.addAll(client.getAllItemsByStatus(status.COMING_SOON));
                        log.info("Before getting paused items.");
                        items.addAll(client.getAllItemsByStatus(status.PAUSED));
                        // Clean up the data from the solr directories.
                        log.info("Before removing old resources.");
                        removeOldResources();
                }

                /*
                 * if (GENERATION_TYPE.equals(GENERATION_TYPE_INCREMENTAL)) { }
                 */
                populateEntityIdItemMap();
                // Generate partners and json objects for phones only
                if (!GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {
                        ProductListGenerator generator = new ProductListGenerator(entityIdItemMap);

                        log.info("Before auto suggest json");
                        synonymTitlesExporter();

                        log.info("Before product list js.");
                        generator.generateProductListJavascript();
                }
                PriceInsertor priceInserter = new PriceInsertor();

                Map<Long, List<String>> entityTags = client.getAllEntityTags();
                Map<Long, Integer> popularityMap = getPolularityMap();
                List<Long> pausedByRiskItems = getRiskyPausedItems();
                for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {
                        long entityId = entry.getKey();
                        List<Item> items = entry.getValue();
                        double minPrice = 0d;

                        // Evaluating availability
                        String availability = "Out of Stock";
                        for (Item i : items) {
                                if (i.getItemStatus().equals(status.ACTIVE)) {
                                        if (!(i.isRisky() && pausedByRiskItems.contains(i.getId()))) {
                                                availability = "In Stock";
                                        }
                                }
                        }
                        StringBuilder priceString = new StringBuilder();
                        StringBuilder availabilityString = new StringBuilder();
                        boolean domainOnce = true;
                        boolean sourceOnce = true;
                        for (String domainPath : DOMAINPATHS) {
                                String domainName = domainPath;
                                String pathName = domainPath.split("\\.")[0].split(":")[0];
                                if (!removeEntities.contains(entityId)) {
                                        priceInserter.insertPriceInHtml(items, entityId, domainName,
                                                        Utils.EXPORT_PATH + "html/entities-" + pathName + "/", null);
                                }
                                if (domainOnce) {
                                        minPrice = getMinPrice(items, entityId, null);
                                        priceString.append("<field name=\"F_50002\">" + minPrice + "</field>");
                                        availabilityString.append("<field name=\"F_50028\">" + availability + "</field>");
                                        if (entityTags.containsKey(entityId)) {
                                                List<String> tags = entityTags.get(entityId);
                                                for (String tag : tags) {
                                                        availabilityString.append("\n<field name=\"F_50029\">" + tag + "</field>");
                                                }
                                        }
                                        if (popularityMap.containsKey(entityId)) {
                                                availabilityString
                                                                .append("\n<field name=\"F_50030\">" + popularityMap.get(entityId) + "</field>");
                                        } else {
                                                availabilityString.append("\n<field name=\"F_50030\">" + "0" + "</field>");
                                        }
                                        domainOnce = false;
                                }
                                if (sources != null) {
                                        for (Source source : sources) {
                                                priceInserter.insertPriceInHtml(items, entityId, domainName,
                                                                Utils.EXPORT_PATH + "html/entities-" + pathName + "/", source);
                                                if (sourceOnce) {
                                                        minPrice = getMinPrice(items, entityId, source);
                                                        priceString
                                                                        .append("<field name=\"F_50002_" + source.getId() + "\">" + minPrice + "</field>");
                                                }
                                        }
                                        sourceOnce = false;
                                }
                        }

                        priceInserter.insertPriceInSolrData(entityId, priceString.toString(), availabilityString.toString());
                }

                priceInserter.copySolrSchemaFiles();
        }

        private Map<Long, Integer> getPolularityMap() {
                try {
                        Reader reader = new FileReader(Utils.EXPORT_PATH + Utils.POPULARITY_JSON);
                        Map<Long, Integer> result = new Gson().fromJson(reader, new TypeToken<Map<Long, Integer>>() {
                        }.getType());
                        if (result == null) {
                                result = new HashMap<Long, Integer>();
                        }
                        return result;
                } catch (Exception e) {
                        log.error("Could not read popularity file");
                        e.printStackTrace();
                        return new HashMap<Long, Integer>();
                }

        }

        private List<Long> getRiskyPausedItems() {
                try {
                        Reader reader = new FileReader(Utils.EXPORT_PATH + Utils.RISKY_PAUSED_JSON);
                        List<Long> result = new Gson().fromJson(reader, new TypeToken<List<Long>>() {
                        }.getType());
                        if (result == null) {
                                result = new ArrayList<Long>();
                        }
                        return result;
                } catch (FileNotFoundException e) {
                        log.error("Could not read paused file");
                        e.printStackTrace();
                        return new ArrayList<Long>();
                }
        }

        /**
     * Generates content for the specified entity embedding links to the
     * specified domain name.
     * 
     * The method will not generate content if one of the following conditions
     * is met:
     * <ol>
     * <li>The entity is not ready.
     * <li>The category has not been updated yet. (Set to -1).
     * <li>The content has not been updated.
     * </ol>
     * 
     * @throws
     */
    private void generateContent() throws Exception {
        if (GENERATION_TYPE.equals(GENERATION_TYPE_ALL)) {
            entities = CreationUtils.getEntities();
            lastGenerationTime = new Long(0);
            items = client.getAllItemsByStatus(status.CONTENT_COMPLETE);
            items.addAll(client.getAllItemsByStatus(status.COMING_SOON));
            //items.addAll(client.getAllItemsByStatus(status.PARTIALLY_ACTIVE));
            items.addAll(client.getAllItemsByStatus(status.ACTIVE));
            items.addAll(client.getAllItemsByStatus(status.PAUSED));
        } else if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {
            entities = new HashMap<Long, Entity>();
            entities.put(Long.parseLong(ENTITY_ID),
                    CreationUtils.getEntity(Long.parseLong(ENTITY_ID)));
            items = client.getItemsByCatalogId(Long.parseLong(ENTITY_ID));
            Iterator<Item> ite = items.iterator();
            while(ite.hasNext()) {
                Item i = ite.next();
                if(!(i.getItemStatus().equals(status.ACTIVE) || i.getItemStatus().equals(status.PAUSED) 
                || i.getItemStatus().equals(status.CONTENT_COMPLETE) || i.getItemStatus().equals(status.COMING_SOON))){
                        ite.remove();
                }
            }
            lastGenerationTime = new Long(0);
        } else {
                entities = CreationUtils.getEntities();
            //  When we read lastGenerationTime from database
            //  then only we should mark the 
            //  current time as newLastGenerationTime
                newLastGenerationTime = new Date().getTime();
            if(timeStamp == null) {
                lastGenerationTime = CreationUtils.getLastContentGenerationTime();
            } else {
                lastGenerationTime = timeStamp.getTime();
            }
            
            log.info("lastGenerationTime: " + lastGenerationTime);
            if (lastGenerationTime == null) {
                lastGenerationTime = new Long(0);
            }
            items = client.getAllItemsByStatus(status.CONTENT_COMPLETE);
            items.addAll(client.getAllItemsByStatus(status.COMING_SOON));
            //items.addAll(client.getAllItemsByStatus(status.PARTIALLY_ACTIVE));
        } 

        // Filter invalid entities here
        List<Entity> validEntities = new ArrayList<Entity>();
        List<Long> validEntityIds = new ArrayList<Long>();
        
        for (long entityID : entities.keySet()) {
            if (isValidEntity(entities.get(entityID))) {
                validEntities.add(entities.get(entityID));
                validEntityIds.add(entityID);
            }
        }
        
        log.info("Generating synonyms");
        SynonymExporter sx = new SynonymExporter();
        sx.storeSynonyms(validEntities);

        // Calculate comparison scores
        log.info("Calculating comparison scores");
        NewCMP cmp = new NewCMP(validEntities);
        Map<Long, Map<Long, Double>> slideScoresByEntity = cmp.getSlideScores();
        CreationUtils.storeSlideScores(slideScoresByEntity);

        // Fetch comparison statistics everyday and store them in BDB
        //This might be remove. 
        log.info("Fetching comparison statistics");
        ComparisonStatsFetcher csf = new ComparisonStatsFetcher();
        csf.fetchAndStoreComparisonStats();
        populateEntityIdItemMap();

        
        log.info("Writing JSON file for special pages");
        SpecialPageJSONConvertor bjc = new SpecialPageJSONConvertor();
        bjc.writeToJSONFile(new File(Utils.EXPORT_JAVASCRIPT_CONTENT_PATH
                + "special-pages.json"));

        log.info("Generating velocity templates, images, documents etc.");
        NewVUI vui = new NewVUI(lastGenerationTime);
        
        for (Entity entity : validEntities) {
            log.info("Processing Entityid: " + entity.getID());
            vui.generateContentForOneEntity(entity, Utils.EXPORT_VELOCITY_PATH);
        }

        try {
                /*AmazonSCDataGenerator ascdg = new AmazonSCDataGenerator(validPartialEntities);*/
                AmazonSCDataGenerator ascdg = new AmazonSCDataGenerator(validPartialEntities,false);
                ascdg.generateSCProdData();
        }catch (Exception e ){
                log.info("Could not generate Jungle upload Feed");
                e.printStackTrace();
        }
        
        if (newLastGenerationTime != 0) {
            CreationUtils.storeLastContentGenerationTime(newLastGenerationTime);
        }


        log.info("Generating Solr files");
        NewIR ir = new NewIR(validEntities);
        ir.exportIRData();
        // ir.transformIrDataXMLtoSolrXML();em.
        
        if(!GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {
                ir.exportIRMetaData();
                ir.transformIrMetaDataXMLtoSolrSchemaXML();
                ir.transformIrMetaDataXMLtoSolrCatchAllXML();
        }
        csc = new CatalogClient();
        client = csc.getClient();
        for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {
            List<Item> items = entry.getValue();
            for (Item item : items) {
                if (item.getItemStatus() == status.CONTENT_COMPLETE 
                                || item.getItemStatus() == status.COMING_SOON) {
                        if(item.getStartDate() <= new Date().getTime() + ONE_DAY){
                                item.setItemStatus(status.ACTIVE);
                                item.setStatus_description("This item is active");
                        } else {
                                item.setItemStatus(status.COMING_SOON);
                                String productName = getProductName(item);
                                String statusDescription = productName + " is coming soon.";
                                if(item.getExpectedArrivalDate()>new Date().getTime() + ONE_DAY){
                                        statusDescription = productName + " will be available by " 
                                        + new SimpleDateFormat("dd/MM/yy").format(new Date(item.getExpectedArrivalDate()));
                                }
                                //Send alert to Category team one day before expected arrival date
                                //So they may change the expected arrival date if they want to.
                                if(item.getExpectedArrivalDate() < new Date().getTime() + 2*ONE_DAY && 
                                                item.getExpectedArrivalDate() > new Date().getTime() + ONE_DAY) {
                                                alertItems.add(item);
                                }
                                item.setStatus_description(statusDescription);
                        }
                        try {
                                client.updateItem(item);
                        }catch( Exception e){
                                csc = new CatalogClient();
                        client = csc.getClient();
                        log.info("Client generated again for item " + item.getId());
                        log.info("Skipped  item " + item.getId());
                        //client.updateItem(item);
                        }
                }
            }
        }
        sendAlertToCategoryTeam(alertItems);
    }

        private void sendAlertToCategoryTeam(List<Item> items) {
                if (items != null && items.size() != 0) {
                        GmailUtils util = new GmailUtils();
                        String[] recipients = { "amit.gupta@shop2020.in", "chaitnaya.vats@shop2020.in",
                                        "khushal.bhatia@shop2020.in", "vrinda.k@shop2020.in" };
                        String from = "build@shop2020.in";
                        String password = "cafe@nes";
                        String subject = Utils.EXPECTED_ARRIVAL_ACHIEVED_TEMPLATE;
                        StringBuffer message = new StringBuffer("Please check the following items:\n");
                        List<File> emptyList = new ArrayList<File>();
                        for (Item item : items) {
                                message.append("\t" + getProductName(item));
                        }
                        try {
                                util.sendSSLMessage(recipients, subject, message.toString(), from, password, emptyList);
                        } catch (Exception e) {
                                log.info("Could not send alert" + e);
                        }
                }
        }

        private String getProductName(Item item) {
                String brand = item.getBrand();
                String modelName = item.getModelName();
                String modelNumber = item.getModelNumber();
                String product = "";
                if (StringUtils.isEmpty(modelName)) {
                        product = brand + " " + modelNumber;
                } else {
                        product = brand + " " + modelName + " " + modelNumber;
                }
                return product;
        }

        /**
         * Checks weather entity is valid or not. Entity will be invalid in one of these
         * cases:
         * <ol>
         * <li>The entity is not ready.
         * <li>The category has not been updated yet. (Set to -1).
         * <li>Content has not been updated after last content generation timestamp.
         * </ol>
         * 
         * @param entity
         * @return
         * @throws Exception
         */
        private boolean isValidEntity(Entity entity) throws Exception {
                EntityState state = CreationUtils.getEntityState(entity.getID());
                long categoryID = state.getCategoryID();
                if (state.getStatus() != EntityStatus.READY || categoryID == -1) {
                        return false;
                }
                if (state.getMerkedReadyOn().getTime() < this.lastGenerationTime) {
                        return false;
                }
                if (state.getMerkedReadyOn().getTime() > this.lastGenerationTimeNonZero) {
                        Utils.info("Added to Partail:" + entity.getID());
                        validPartialEntities.add(entity);
                }
                return true;
        }

        private void populateEntityIdItemMap() {
                Date todate = new Date();
                Utils.info("Processing " + items.size() + " items");
                for (Item item : items) {
                        Utils.info(item.getId() + ":" + item.getItemStatus() + ":" + item.getCatalogItemId());
                        // TODO Can be removed as we are checking in calling function
                        /*
                         * if (!(item.getItemStatus() == status.ACTIVE || item.getItemStatus() ==
                         * status.CONTENT_COMPLETE || item .getItemStatus() == status.PAUSED ||
                         * item.getItemStatus() == status.COMING_SOON)) { continue; }
                         */
                        SimpleDateFormat df1 = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
                        Utils.info(df1.format(item.getStartDate()) + ":" + item.getSellingPrice());

                        if (todate.getTime() < item.getStartDate()
                                        && (!item.isSetExpectedArrivalDate() || todate.getTime() < item.getComingSoonStartDate())
                                        || item.getSellingPrice() == 0) {
                                continue;
                        }
                        Utils.info(item.getId() + " Item is adding");
                        List<Item> itemList = entityIdItemMap.get(item.getCatalogItemId());
                        if (itemList == null) {
                                itemList = new ArrayList<Item>();
                        }
                        itemList.add(item);
                        entityIdItemMap.put(item.getCatalogItemId(), itemList);
                }

                Utils.info("Processing " + entityIdItemMap.size() + " entities");
                // Remove all items which have not been updated since last content
                // generation.
                if (!(UPDATE_TYPE_CONTENT.equals(UPDATE_TYPE) || lastGenerationTime == 0)) {
                        for (Long entityId : entityIdItemMap.keySet()) {
                                boolean isValidEntity = false;
                                // If any one of the items has been updated before current
                                // timestamp, than we generate content for pricing
                                for (Item item : entityIdItemMap.get(entityId)) {
                                        if (item.getUpdatedOn() > lastGenerationTime) {
                                                isValidEntity = true;
                                        }
                                }
                                if (!isValidEntity) {
                                        removeEntities.add(entityId);
                                }
                        }
                }

                Utils.info("Final valid entities to be processed: " + entityIdItemMap.size());
        }

        private void synonymTitlesExporter() {
                SynonymExporter sx = new SynonymExporter();
                Map<Long, Map<String, List<String>>> synonyms = sx.getSynonyms();
                Map<String, List<String>> finalsynonyms = new HashMap<String, List<String>>();
                for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {
                        long entityId = entry.getKey();
                        try {
                                String brand = "";
                                String originalModelName = "";
                                String originalModelNumber = "";
                                List<String> modelNameSynonyms = new ArrayList<String>();
                                List<String> modelNumberSynonyms = new ArrayList<String>();
                                List<String> titles = new ArrayList<String>();
                                Map<String, List<String>> synonymMap = synonyms.get(entityId);
                                if (synonymMap != null && !synonymMap.isEmpty()) {
                                        if (synonymMap.get("ORIGINAL_MODEL_NAME") != null
                                                        && !synonymMap.get("ORIGINAL_MODEL_NAME").isEmpty()) {
                                                modelNameSynonyms.addAll(synonymMap.get("ORIGINAL_MODEL_NAME"));
                                                originalModelName = synonymMap.get("ORIGINAL_MODEL_NAME").get(0);
                                        }
                                        if (synonymMap.get("MODEL_NAME") != null && !synonymMap.get("MODEL_NAME").isEmpty()) {
                                                modelNameSynonyms.addAll(synonymMap.get("MODEL_NAME"));
                                        }
                                        if (synonymMap.get("ORIGINAL_MODEL_NUMBER") != null
                                                        && !synonymMap.get("ORIGINAL_MODEL_NUMBER").isEmpty()) {
                                                modelNumberSynonyms.addAll(synonymMap.get("ORIGINAL_MODEL_NUMBER"));
                                                originalModelNumber = synonymMap.get("ORIGINAL_MODEL_NUMBER").get(0);
                                        }
                                        if (synonymMap.get("MODEL_NUMBER") != null && !synonymMap.get("MODEL_NUMBER").isEmpty()) {
                                                modelNumberSynonyms.addAll(synonymMap.get("MODEL_NUMBER"));
                                        }
                                        brand = ((synonymMap.get("ORIGINAL_BRAND") != null && !synonymMap.get("ORIGINAL_BRAND").isEmpty())
                                                        ? synonymMap.get("ORIGINAL_BRAND").get(0)
                                                        : "");
                                }
                                for (String model_name : modelNameSynonyms) {
                                        for (String model_number : modelNumberSynonyms) {
                                                String title = brand + " " + model_name + " " + model_number;
                                                title = title.replaceAll("  ", " ");
                                                titles.add(title);
                                        }
                                }
                                String originaltitle = brand + " " + originalModelName + " " + originalModelNumber;
                                originaltitle = originaltitle.replaceAll("  ", " ");
                                originaltitle = originaltitle.trim();
                                if (!originaltitle.isEmpty()) {
                                        finalsynonyms.put(originaltitle, titles);
                                }
                        } catch (Exception e) {
                                e.printStackTrace();
                        }
                }

                String autosuggestFilename = Utils.EXPORT_JAVASCRIPT_CONTENT_PATH + "autosuggest.json";
                Gson gson = new Gson();
                try {
                        DBUtils.store(gson.toJson(finalsynonyms), autosuggestFilename);
                } catch (Exception e) {
                        e.printStackTrace();
                }
        }

        public double getMinPrice(List<Item> items, long catalogId, Source source) {
                Item minPriceItem = null;
                for (Item item : items) {
                        if (minPriceItem == null || minPriceItem.getSellingPrice() > item.getSellingPrice()) {
                                minPriceItem = item;
                        }
                }
                return minPriceItem.getSellingPrice();
        }
}