Subversion Repositories SmartDukaan

package in.shop2020.util;

import in.shop2020.metamodel.core.Entity;
import in.shop2020.metamodel.core.EntityState;
import in.shop2020.metamodel.core.EntityStatus;
import in.shop2020.metamodel.util.CreationUtils;
import in.shop2020.model.v1.catalog.CatalogService.Client;
import in.shop2020.model.v1.catalog.Item;
import in.shop2020.model.v1.catalog.Source;
import in.shop2020.model.v1.catalog.status;
import in.shop2020.thrift.clients.CatalogClient;
import in.shop2020.ui.util.ComparisonStatsFetcher;
import in.shop2020.ui.util.NewVUI;
import in.shop2020.ui.util.PriceInsertor;
import in.shop2020.ui.util.SpecialPageJSONConvertor;
import in.shop2020.utils.GmailUtils;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

import javax.mail.MessagingException;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;

public class ContentGenerationUtility {
    private static final String   UPDATE_TYPE_CATALOG         = "CATALOG";

    private static final String   UPDATE_TYPE_CONTENT         = "CONTENT";

    private static final String   COMMAND_LINE                = "java ContentGenerationUtility.class -t { ALL | INCREMENTAL | ONE } -s { yyyy-MM-dd-HH-mm-ss } -u { CONTENT | CATALOG } -e {EntityId} ";

    private static Log            log                         = LogFactory
                                                                      .getLog(ContentGenerationUtility.class);
    private static long                         ONE_DAY                                   = 24*60*60*1000;      //milliseconds in a day                                                 
    // Commandline options
    private static Options        options                     = null;
    private static final String   GENERATION_TYPE_INCREMENTAL = "INCREMENTAL";
    private static final String   GENERATION_TYPE_ALL         = "ALL";
    private static final String   GENERATION_TYPE_ONE         = "ONE";
    private static final String   UPDATE_TYPE_OPTION          = "u";
    private static final String   GENERATION_TYPE_OPTION      = "t";
    private static final String   ENTITY_ID_OPTION            = "e";
    private static final String   TIMESTAMP_OPTION            = "s";

    // Default values of cmdline options
    private static String         UPDATE_TYPE                 = UPDATE_TYPE_CONTENT;
    private static String         GENERATION_TYPE             = GENERATION_TYPE_INCREMENTAL;
    private static String         ENTITY_ID                   = "ALL";
    private static String []      DOMAINPATHS                         = Utils.DOMAIN_NAMES_FOR_CONTENT_GENERATION.split(";");

    private Date                                  timeStamp                                       = null;
    private CommandLine           cmd                         = null;
    private Map<Long, List<Item>> entityIdItemMap             = new LinkedHashMap<Long, List<Item>>();
    private Long                  lastGenerationTime          = 0l;
    private Long                  lastGenerationTimeNonZero          = CreationUtils.getLastContentGenerationTime();
    private Map<Long, Entity>     entities;
    private List<Long> removeEntities = new ArrayList<Long>();
    private List<Item>            items;
    private List<Source>          sources;
    private CatalogClient         csc;
    List<Entity> validPartialEntities = new ArrayList<Entity>();
    private Client                client;
    private List<Item>                    alertItems;   
    private long                  newLastGenerationTime;

    static {
        options = new Options();
        options.addOption(GENERATION_TYPE_OPTION, true, "Generation type");
        options.addOption(UPDATE_TYPE_OPTION, true, "Default is : "
                + UPDATE_TYPE);
        options.addOption(ENTITY_ID_OPTION, true, "all entities " + ENTITY_ID
                + " by default");
        options.addOption(TIMESTAMP_OPTION, true, "Manual timestamp");

    }

    public ContentGenerationUtility() throws Exception {
        csc = new CatalogClient();
        client = csc.getClient();
        sources = client.getAllSources();
        alertItems = new ArrayList<Item>();     
    }

    /**
     * @param args
     * @throws Exception
     */
    public static void main(String[] args) throws Exception {
        ContentGenerationUtility cgu = new ContentGenerationUtility();

        // Load arguments
        cgu.loadArgs(args);

        // Call method based on arguments
        cgu.callMethod();
    }

    /**
     * Validate and set command line arguments. Exit after printing usage if
     * anything is astray
     * 
     * @param args
     *            String[] args as featured in public static void main()
     */
    private void loadArgs(String[] args) {
        CommandLineParser parser = new PosixParser();

        try {
            cmd = parser.parse(options, args);
        } catch (ParseException e) {
            log.error("Error parsing arguments", e);
            System.exit(1);
        }

        // Check for mandatory args
        if (!(cmd.hasOption(GENERATION_TYPE_OPTION) && cmd
                .hasOption(UPDATE_TYPE_OPTION))) {
            HelpFormatter formatter = new HelpFormatter();
            formatter.printHelp(COMMAND_LINE, options);
            System.exit(1);
        }
        
        
        GENERATION_TYPE = cmd.getOptionValue(GENERATION_TYPE_OPTION);
        
        UPDATE_TYPE = cmd.getOptionValue(UPDATE_TYPE_OPTION);

        // Look for optional args.
        if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {
            if (cmd.hasOption(ENTITY_ID_OPTION)) {
                ENTITY_ID = cmd.getOptionValue(ENTITY_ID_OPTION);
            } else {
                HelpFormatter formatter = new HelpFormatter();
                formatter.printHelp(COMMAND_LINE, options);
                System.exit(1);
            }
        }
        
        if (GENERATION_TYPE_INCREMENTAL.equals(GENERATION_TYPE))
                if (cmd.hasOption(TIMESTAMP_OPTION)) {
                        SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
                        try {
                            timeStamp = df.parse(cmd.getOptionValue(TIMESTAMP_OPTION));
                        } catch(Exception e) {
                                HelpFormatter formatter = new HelpFormatter();
                    formatter.printHelp(COMMAND_LINE, options);
                    System.exit(1);
                        }
                }
    }

    /**
     * Call method based on arguments
     * 
     * @throws Exception
     */
    private void callMethod() {
        boolean isSuccess = false;
        String logfile = "/tmp/content-from-cms.log";
        if (UPDATE_TYPE.equals(UPDATE_TYPE_CONTENT)) {
            logfile = "/tmp/content-from-cms.log";
            try {
                this.generateContent();
                isSuccess = true;
            } catch (Exception e) {
                log.error("Error generating content", e);
            }
        }

        if (UPDATE_TYPE.equals(UPDATE_TYPE_CATALOG)) {
            logfile = "/tmp/content-from-catalog.log";
            try {
                this.updatePrices();
                isSuccess = true;
            } catch (Exception e) {
                log.error("Error updating prices", e);
            }
        }

        GmailUtils gm = new GmailUtils();
        String[] sendTo = { "rajveer.singh@shop2020.in", "anupam.singh@shop2020.in", "amit.gupta@shop2020.in" };

        try {
            gm.sendSSLMessage(sendTo, "Content Generation Successful ? : "
                    + isSuccess, "Content generation completed at time : "
                    + Calendar.getInstance().getTime().toString(),
                    "build@shop2020.in", "cafe@nes", logfile);
        } catch (MessagingException e) {
            log.error("Could not send status mail", e);
        }
    }

    public boolean cleanDir(File dir, boolean deleteSelf) {
        if (dir.isDirectory()) {
            String[] children = dir.list();
            for (int i = 0; i < children.length; i++) {
                boolean success = cleanDir(new File(dir, children[i]), true);
                if (!success) {
                    return false;
                }
            }
        }

        // The directory is now empty so delete it
        if (deleteSelf) {
            return dir.delete();
        }

        return true;
    }

    private void removeOldResources() throws IOException {
        File f = new File(Utils.EXPORT_SOLR_PATH);
        if (f.exists()) {
            cleanDir(f, false);
        }
        
        for(String domainPath : DOMAINPATHS){
                String pathName = domainPath.split("\\.")[0].split(":")[0];
                File f1 = new File(Utils.EXPORT_PATH + "html/entities-" +  pathName);
                if (f1.exists()) {
                        cleanDir(f1, false);
                }else {
                        f1.mkdir();
                }
        }
    }

    /**
     * Update the prices in the generated content
     * 
     * @throws Exception
     */
    private void updatePrices() throws Exception {
        lastGenerationTime = new Long(0);
        List<Long> activeItems = new ArrayList<Long>();
        if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {
            items = client.getItemsByCatalogId(Long.parseLong(ENTITY_ID));
            Iterator<Item> it = items.iterator();
            while(it.hasNext()){
                Item ite = it.next();
                status st = ite.getItemStatus();
                if(!(st.equals(status.ACTIVE) || st.equals(status.PAUSED) || 
                                        st.equals(status.COMING_SOON))){
                        it.remove();
                }
                if(st.equals(status.ACTIVE)){
                        activeItems.add(ite.getId());
                }
            }
            try {
                //Generate prices and availability data for amazon
                AmazonSCDataGenerator.generatePricesAndAvailability(items);
            } catch (Exception e) {
                log.info("Could not generate Amazon prices and availability", e);
            }
            //ProductListGenerator.updatePriceForEntity(Long.parseLong(ENTITY_ID), items.get(0).getSellingPrice(), items.get(0).getMrp());
        } else {
                log.info("Before getting active items.");
            items = client.getAllItemsByStatus(status.ACTIVE);
            for (Item item : items) {
                activeItems.add(item.getId());
            }
            log.info("Before getting coming items.");
            items.addAll(client.getAllItemsByStatus(status.COMING_SOON));
            log.info("Before getting paused items.");
            items.addAll(client.getAllItemsByStatus(status.PAUSED));
            // Clean up the data from the solr directories.
            log.info("Before removing old resources.");
            removeOldResources();
        }

/*        if (GENERATION_TYPE.equals(GENERATION_TYPE_INCREMENTAL)) {
        }
*/
        populateEntityIdItemMap();
        // Generate partners and json objects for phones only
        if (!GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {
                ProductListGenerator generator = new ProductListGenerator(entityIdItemMap);
                
                log.info("Before auto suggest json");
                synonymTitlesExporter();

                log.info("Before product list js.");
                generator.generateProductListJavascript(); 
        }
        PriceInsertor priceInserter = new PriceInsertor();

        Map<Long,List<String>> entityTags = client.getAllEntityTags();
        Map<Long, Integer> popularityMap = getPolularityMap();
        List<Long> pausedByRiskItems = getRiskyPausedItems();
        for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {
            long entityId = entry.getKey();
            List<Item> items = entry.getValue();
            double minPrice = 0d;
            
            //Evaluating availability
            String availability = "Out of Stock";
            for(Item i: items){
                if(i.getItemStatus().equals(status.ACTIVE)) {
                        if(!(i.isRisky() && pausedByRiskItems.contains(i.getId()) )){
                                availability = "In Stock";
                        }
                }
            }
            StringBuilder priceString = new StringBuilder();
            StringBuilder availabilityString = new StringBuilder();
            boolean domainOnce = true;
            boolean sourceOnce = true;
            for(String domainPath : DOMAINPATHS){
                String domainName = domainPath;
                String pathName = domainPath.split("\\.")[0].split(":")[0];
                if(!removeEntities.contains(entityId)){
                        priceInserter.insertPriceInHtml(items, entityId, domainName, Utils.EXPORT_PATH + "html/entities-" +  pathName + "/", null);
                }
                if(domainOnce){
                        minPrice = getMinPrice(items, entityId, null);
                        priceString.append("<field name=\"F_50002\">" + minPrice + "</field>");
                        availabilityString.append("<field name=\"F_50028\">" + availability + "</field>");
                        if(entityTags.containsKey(entityId)) {
                                List<String> tags = entityTags.get(entityId);
                                for(String tag: tags){
                                        availabilityString.append("\n<field name=\"F_50029\">" + tag + "</field>");
                                }
                        }
                        if(popularityMap.containsKey(entityId)) {
                                availabilityString.append("\n<field name=\"F_50030\">" + popularityMap.get(entityId) + "</field>");
                        }else {
                                availabilityString.append("\n<field name=\"F_50030\">" + "0" + "</field>");
                        }
                        domainOnce = false;
                }
                if(sources != null){
                        for (Source source : sources) {
                                                priceInserter.insertPriceInHtml(items, entityId,domainName, Utils.EXPORT_PATH + "html/entities-" + pathName + "/",source);
                        if(sourceOnce){
                                minPrice = getMinPrice(items, entityId, source);
                                priceString.append("<field name=\"F_50002_"
                                + source.getId() + "\">" + minPrice + "</field>");
                        }
                        }
                        sourceOnce = false;
                }
            }

            priceInserter.insertPriceInSolrData(entityId,
                    priceString.toString(), availabilityString.toString());
        }

        priceInserter.copySolrSchemaFiles();
    }

        private Map<Long, Integer> getPolularityMap() {
                try {
                        Reader reader = new FileReader(Utils.EXPORT_PATH + Utils.POPULARITY_JSON);
                        return new Gson().fromJson(reader, new TypeToken<Map<Long, Integer>>() {}.getType());
                } catch (Exception e) {
                        log.error("Could not read popularity file");
                        e.printStackTrace();
                        return new HashMap<Long, Integer>();
                }
        
        }
        private List<Long> getRiskyPausedItems() {
                try {
                        Reader reader = new FileReader(Utils.EXPORT_PATH + Utils.RISKY_PAUSED_JSON);
                        return new Gson().fromJson(reader, new TypeToken<List<Long>>() {}.getType());
                } catch (FileNotFoundException e) {
                        log.error("Could not read paused file");
                        e.printStackTrace();
                        return new ArrayList<Long>();
                }
        }

        /**
     * Generates content for the specified entity embedding links to the
     * specified domain name.
     * 
     * The method will not generate content if one of the following conditions
     * is met:
     * <ol>
     * <li>The entity is not ready.
     * <li>The category has not been updated yet. (Set to -1).
     * <li>The content has not been updated.
     * </ol>
     * 
     * @throws
     */
    private void generateContent() throws Exception {
        if (GENERATION_TYPE.equals(GENERATION_TYPE_ALL)) {
            entities = CreationUtils.getEntities();
            lastGenerationTime = new Long(0);
            items = client.getAllItemsByStatus(status.CONTENT_COMPLETE);
            items.addAll(client.getAllItemsByStatus(status.COMING_SOON));
            items.addAll(client.getAllItemsByStatus(status.ACTIVE));
            items.addAll(client.getAllItemsByStatus(status.PAUSED));
        } else if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {
            entities = new HashMap<Long, Entity>();
            entities.put(Long.parseLong(ENTITY_ID),
                    CreationUtils.getEntity(Long.parseLong(ENTITY_ID)));
            items = client.getItemsByCatalogId(Long.parseLong(ENTITY_ID));
            Iterator<Item> ite = items.iterator();
            while(ite.hasNext()) {
                Item i = ite.next();
                if(!(i.getItemStatus().equals(status.ACTIVE) || i.getItemStatus().equals(status.PAUSED) 
                || i.getItemStatus().equals(status.CONTENT_COMPLETE) || i.getItemStatus().equals(status.COMING_SOON))){
                        ite.remove();
                }
            }
            lastGenerationTime = new Long(0);
        } else {
                entities = CreationUtils.getEntities();
            //  When we read lastGenerationTime from database
            //  then only we should mark the 
            //  current time as newLastGenerationTime
            if(timeStamp == null) {
                newLastGenerationTime = new Date().getTime();
                lastGenerationTime = CreationUtils.getLastContentGenerationTime();
            } else {
                lastGenerationTime = timeStamp.getTime();
            }
            
            log.info("lastGenerationTime: " + lastGenerationTime);
            if (lastGenerationTime == null) {
                lastGenerationTime = new Long(0);
            }
            items = client.getAllItemsByStatus(status.CONTENT_COMPLETE);
            items.addAll(client.getAllItemsByStatus(status.COMING_SOON));
        } 

        // Filter invalid entities here
        List<Entity> validEntities = new ArrayList<Entity>();
        List<Long> validEntityIds = new ArrayList<Long>();
        
        for (long entityID : entities.keySet()) {
            if (isValidEntity(entities.get(entityID))) {
                validEntities.add(entities.get(entityID));
                validEntityIds.add(entityID);
            }
        }
        
        log.info("Generating synonyms");
        SynonymExporter sx = new SynonymExporter();
        sx.storeSynonyms(validEntities);

        // Calculate comparison scores
        log.info("Calculating comparison scores");
        NewCMP cmp = new NewCMP(validEntities);
        Map<Long, Map<Long, Double>> slideScoresByEntity = cmp.getSlideScores();
        CreationUtils.storeSlideScores(slideScoresByEntity);

        // Fetch comparison statistics everyday and store them in BDB
        //This might be remove. 
        log.info("Fetching comparison statistics");
        ComparisonStatsFetcher csf = new ComparisonStatsFetcher();
        csf.fetchAndStoreComparisonStats();
        populateEntityIdItemMap();

        
        log.info("Writing JSON file for special pages");
        SpecialPageJSONConvertor bjc = new SpecialPageJSONConvertor();
        bjc.writeToJSONFile(new File(Utils.EXPORT_JAVASCRIPT_CONTENT_PATH
                + "special-pages.json"));

        log.info("Generating velocity templates, images, documents etc.");
        NewVUI vui = new NewVUI(lastGenerationTime);
        
        for (Entity entity : validEntities) {
            log.info("Processing Entityid: " + entity.getID());
            vui.generateContentForOneEntity(entity, Utils.EXPORT_VELOCITY_PATH);
        }

        try {
                AmazonSCDataGenerator ascdg = new AmazonSCDataGenerator(validPartialEntities);
                ascdg.generateSCProdData();
        }catch (Exception e ){
                log.info("Could not generate Jungle upload Feed");
                e.printStackTrace();
        }
        
        if (newLastGenerationTime != 0) {
            CreationUtils.storeLastContentGenerationTime(newLastGenerationTime);
        }


        log.info("Generating Solr files");
        NewIR ir = new NewIR(validEntities);
        ir.exportIRData();
        // ir.transformIrDataXMLtoSolrXML();
        
        if(!GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {
                ir.exportIRMetaData();
                ir.transformIrMetaDataXMLtoSolrSchemaXML();
                ir.transformIrMetaDataXMLtoSolrCatchAllXML();
        }

        for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {
            List<Item> items = entry.getValue();
            for (Item item : items) {
                if (item.getItemStatus() == status.CONTENT_COMPLETE 
                                || item.getItemStatus() == status.COMING_SOON) {
                        if(item.getStartDate() <= new Date().getTime() + ONE_DAY){
                                item.setItemStatus(status.ACTIVE);
                                item.setStatus_description("This item is active");
                        } else {
                                item.setItemStatus(status.COMING_SOON);
                                String productName = getProductName(item);
                                String statusDescription = productName + " is coming soon.";
                                if(item.getExpectedArrivalDate()>new Date().getTime() + ONE_DAY){
                                        statusDescription = productName + " will be available by " 
                                        + new SimpleDateFormat("dd/MM/yy").format(new Date(item.getExpectedArrivalDate()));
                                }
                                //Send alert to Category team one day before expected arrival date
                                //So they may change the expected arrival date if they want to.
                                if(item.getExpectedArrivalDate() < new Date().getTime() + 2*ONE_DAY && 
                                                item.getExpectedArrivalDate() > new Date().getTime() + ONE_DAY) {
                                                alertItems.add(item);
                                }
                                item.setStatus_description(statusDescription);
                        }
                    client.updateItem(item);
                }
            }
        }
        sendAlertToCategoryTeam(alertItems);
    }

    private void sendAlertToCategoryTeam(List<Item> items) {
        if(items!=null && items.size()!=0){
                        GmailUtils util = new GmailUtils();
                        String[] recipients = {"amit.gupta@shop2020.in", "chaitnaya.vats@shop2020.in", "khushal.bhatia@shop2020.in", "vrinda.k@shop2020.in"};
                        String from = "build@shop2020.in";
                        String password = "cafe@nes";
                        String subject = Utils.EXPECTED_ARRIVAL_ACHIEVED_TEMPLATE;
                        StringBuffer message = new StringBuffer("Please check the following items:\n");
                        List<File> emptyList = new ArrayList<File>();
                        for( Item item : items){
                                message.append("\t" + getProductName(item));
                        }
                        try {
                                util.sendSSLMessage(recipients, subject, message.toString(), from, password, emptyList);
                        } catch (Exception e){
                                log.info("Could not send alert" + e);
                        }
        }
        }

        private String getProductName(Item item) {
        String brand = item.getBrand();
                String modelName = item.getModelName();
                String modelNumber = item.getModelNumber();
                String product = "";
                if(StringUtils.isEmpty(modelName)){
                        product = brand + " " + modelNumber;
                }else {
                        product = brand + " " + modelName + " " + modelNumber;
                }
                return product;
        }

    /**
     * Checks weather entity is valid or not. Entity will be invalid in one of
     * these cases:
     * <ol>
     * <li>The entity is not ready.
     * <li>The category has not been updated yet. (Set to -1).
     * <li>Content has not been updated after last content generation timestamp.
     * </ol>
     * 
     * @param entity
     * @return
     * @throws Exception
     */
    private boolean isValidEntity(Entity entity) throws Exception {
        EntityState state = CreationUtils.getEntityState(entity.getID());
        long categoryID = state.getCategoryID();
        if (state.getStatus() != EntityStatus.READY || categoryID == -1) {
            return false;
        }
        if(state.getMerkedReadyOn().getTime() < this.lastGenerationTime) {
            return false;
        }
        if(state.getMerkedReadyOn().getTime() > this.lastGenerationTimeNonZero){
                        Utils.info("Added to Partail:" + entity.getID());
                        validPartialEntities.add(entity);
        }
        return true;
    }

    private void populateEntityIdItemMap() {
        Date todate = new Date();
        Utils.info("Processing " + items.size() + " items");
        for (Item item : items) {
            Utils.info(item.getId() + ":" + item.getItemStatus() + ":" + item.getCatalogItemId());
            // TODO Can be removed as we are checking in calling function
            if (!(item.getItemStatus() == status.ACTIVE
                    || item.getItemStatus() == status.CONTENT_COMPLETE || item
                    .getItemStatus() == status.PAUSED || item.getItemStatus() == status.COMING_SOON)) {
                continue;
            }
            SimpleDateFormat df1 = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
            Utils.info(df1.format(item.getStartDate()) + ":" + item.getSellingPrice());
            
                        if (todate.getTime() < item.getStartDate()
                                        && (!item.isSetExpectedArrivalDate() || todate.getTime() < item.getComingSoonStartDate()) 
                                        || item.getSellingPrice() == 0) {
                                continue;
                        }
            Utils.info(item.getId() + " Item is adding");
            List<Item> itemList = entityIdItemMap.get(item.getCatalogItemId());
            if (itemList == null) {
                itemList = new ArrayList<Item>();
            } 
            itemList.add(item);
            entityIdItemMap.put(item.getCatalogItemId(), itemList);
        }

        Utils.info("Processing " + entityIdItemMap.size() + " entities");
        // Remove all items which have not been updated since last content
        // generation.
        if (!(UPDATE_TYPE_CONTENT.equals(UPDATE_TYPE) || lastGenerationTime == 0)) {
                for (Long entityId : entityIdItemMap.keySet()) {
                    boolean isValidEntity = false;
                    // If any one of the items has been updated before current
                    // timestamp, than we generate content for pricing
                    for (Item item : entityIdItemMap.get(entityId)) {
                        if (item.getUpdatedOn() > lastGenerationTime) {
                            isValidEntity = true;
                        }
                    }
                    if (!isValidEntity) {
                        removeEntities.add(entityId);
                    }
                }
        }

        Utils.info("Final valid entities to be processed: " + entityIdItemMap.size());
    }

    private void synonymTitlesExporter() {
        SynonymExporter sx = new SynonymExporter();
        Map<Long, Map<String,List<String>>> synonyms = sx.getSynonyms();
        Map<String, List<String>> finalsynonyms = new HashMap<String, List<String>>();
        for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {
            long entityId = entry.getKey();
                try{
                    String brand = "";
                    String originalModelName = "";
                    String originalModelNumber = "";
                    List<String> modelNameSynonyms =  new ArrayList<String>();
                    List<String> modelNumberSynonyms =  new ArrayList<String>();
                    List<String> titles = new ArrayList<String>();
                    Map<String,List<String>> synonymMap = synonyms.get(entityId);
                    if(synonymMap != null && !synonymMap.isEmpty()){
                        if(synonymMap.get("ORIGINAL_MODEL_NAME") != null && !synonymMap.get("ORIGINAL_MODEL_NAME").isEmpty()){
                                modelNameSynonyms.addAll(synonymMap.get("ORIGINAL_MODEL_NAME"));
                                originalModelName = synonymMap.get("ORIGINAL_MODEL_NAME").get(0);
                        }
                        if(synonymMap.get("MODEL_NAME") != null && !synonymMap.get("MODEL_NAME").isEmpty()){
                                modelNameSynonyms.addAll(synonymMap.get("MODEL_NAME"));
                        }
                        if(synonymMap.get("ORIGINAL_MODEL_NUMBER") != null && !synonymMap.get("ORIGINAL_MODEL_NUMBER").isEmpty()){
                                modelNumberSynonyms.addAll(synonymMap.get("ORIGINAL_MODEL_NUMBER"));
                                originalModelNumber = synonymMap.get("ORIGINAL_MODEL_NUMBER").get(0);
                        }
                        if(synonymMap.get("MODEL_NUMBER") != null && !synonymMap.get("MODEL_NUMBER").isEmpty()){
                                modelNumberSynonyms.addAll(synonymMap.get("MODEL_NUMBER"));
                        }
                        brand = ((synonymMap.get("ORIGINAL_BRAND") != null && !synonymMap.get("ORIGINAL_BRAND").isEmpty()) ? synonymMap.get("ORIGINAL_BRAND").get(0) : "");
                    }
                    for(String model_name: modelNameSynonyms){
                        for(String model_number: modelNumberSynonyms){
                                String title = brand + " " + model_name + " " + model_number;
                                title = title.replaceAll("  ", " ");
                                titles.add(title);
                        }
                    }
                    String originaltitle = brand + " " + originalModelName + " " + originalModelNumber;
                    originaltitle = originaltitle.replaceAll("  ", " ");
                    originaltitle = originaltitle.trim();
                    if(!originaltitle.isEmpty()) {
                        finalsynonyms.put(originaltitle, titles);
                    }
                } catch (Exception e) {
                                e.printStackTrace();
                        }
        }
        
        String autosuggestFilename = Utils.EXPORT_JAVASCRIPT_CONTENT_PATH + "autosuggest.json";
        Gson gson = new Gson();
                try {
                        DBUtils.store(gson.toJson(finalsynonyms), autosuggestFilename);
                } catch (Exception e) {
                        e.printStackTrace();
                }
    }

        public double getMinPrice(List<Item> items, long catalogId, Source source) {
                Item minPriceItem = null;
                for (Item item : items) {
                        if (minPriceItem == null
                                        || minPriceItem.getSellingPrice() > item.getSellingPrice()) {
                                minPriceItem = item;
                        }
                }
                return minPriceItem.getSellingPrice();
        }
}