Subversion Repositories SmartDukaan

package in.shop2020.util;

import in.shop2020.metamodel.core.Entity;
import in.shop2020.metamodel.core.EntityState;
import in.shop2020.metamodel.core.EntityStatus;
import in.shop2020.metamodel.util.CreationUtils;
import in.shop2020.metamodel.util.ExpandedEntity;
import in.shop2020.model.v1.catalog.CatalogService.Client;
import in.shop2020.model.v1.catalog.Item;
import in.shop2020.model.v1.catalog.ItemShippingInfo;
import in.shop2020.model.v1.catalog.Source;
import in.shop2020.model.v1.catalog.status;
import in.shop2020.thrift.clients.CatalogClient;
import in.shop2020.ui.util.CatalogUploderToGAE;
import in.shop2020.ui.util.ComparisonStatsFetcher;
import in.shop2020.ui.util.NewVUI;
import in.shop2020.ui.util.PriceInsertor;
import in.shop2020.ui.util.SpecialPageJSONConvertor;
import in.shop2020.utils.ConfigClientKeys;
import in.shop2020.utils.GmailUtils;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

import javax.mail.MessagingException;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.thrift.transport.TTransportException;

import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;

public class ContentGenerationUtility {
    private static final String   UPDATE_TYPE_CATALOG         = "CATALOG";

    private static final String   UPDATE_TYPE_CONTENT         = "CONTENT";

    private static final String   COMMAND_LINE                = "java ContentGenerationUtility.class -t { ALL | INCREMENTAL | ONE } -s { yyyy-MM-dd-HH-mm-ss } -u { CONTENT | CATALOG } -e {EntityId} ";

    private static Log            log                         = LogFactory
                                                                      .getLog(ContentGenerationUtility.class);
    private static long                         ONE_DAY                                   = 24*60*60*1000;      //milliseconds in a day                                                 
    // Commandline options
    private static Options        options                     = null;
    private static final String   GENERATION_TYPE_INCREMENTAL = "INCREMENTAL";
    private static final String   GENERATION_TYPE_ALL         = "ALL";
    private static final String   GENERATION_TYPE_ONE         = "ONE";
    private static final String   UPDATE_TYPE_OPTION          = "u";
    private static final String   GENERATION_TYPE_OPTION      = "t";
    private static final String   ENTITY_ID_OPTION            = "e";
    private static final String   TIMESTAMP_OPTION            = "s";

    // Default values of cmdline options
    private static String         UPDATE_TYPE                 = UPDATE_TYPE_CONTENT;
    private static String         GENERATION_TYPE             = GENERATION_TYPE_INCREMENTAL;
    private static String         ENTITY_ID                   = "ALL";
    private static String []      DOMAINPATHS                         = Utils.DOMAIN_NAMES_FOR_CONTENT_GENERATION.split(";");

    private Date                                  timeStamp                                       = null;
    private CommandLine           cmd                         = null;
    private Map<Long, List<Item>> entityIdItemMap             = new LinkedHashMap<Long, List<Item>>();
    private List<Long>                    allValidEntityIds                       = null;
    private Long                  lastGenerationTime          = 0l;
    private Map<Long, Entity>     entities;
    private List<Item>            items;
    private List<Source>          sources;
    private CatalogClient         csc;
    private Client                client;
    private List<Item>                    alertItems;   
    private long                  newLastGenerationTime;

    static {
        options = new Options();
        options.addOption(GENERATION_TYPE_OPTION, true, "Generation type");
        options.addOption(UPDATE_TYPE_OPTION, true, "Default is : "
                + UPDATE_TYPE);
        options.addOption(ENTITY_ID_OPTION, true, "all entities " + ENTITY_ID
                + " by default");
        options.addOption(TIMESTAMP_OPTION, true, "Manual timestamp");

    }

    public ContentGenerationUtility() throws Exception {
        csc = new CatalogClient();
        client = csc.getClient();
        sources = client.getAllSources();
        alertItems = new ArrayList<Item>();     
    }

    /**
     * @param args
     * @throws Exception
     */
    public static void main(String[] args) throws Exception {
        ContentGenerationUtility cgu = new ContentGenerationUtility();

        // Load arguments
        cgu.loadArgs(args);

        // Call method based on arguments
        cgu.callMethod();
    }

    /**
     * Validate and set command line arguments. Exit after printing usage if
     * anything is astray
     * 
     * @param args
     *            String[] args as featured in public static void main()
     */
    private void loadArgs(String[] args) {
        CommandLineParser parser = new PosixParser();

        try {
            cmd = parser.parse(options, args);
        } catch (ParseException e) {
            log.error("Error parsing arguments", e);
            System.exit(1);
        }

        // Check for mandatory args
        if (!(cmd.hasOption(GENERATION_TYPE_OPTION) && cmd
                .hasOption(UPDATE_TYPE_OPTION))) {
            HelpFormatter formatter = new HelpFormatter();
            formatter.printHelp(COMMAND_LINE, options);
            System.exit(1);
        }
        
        
        GENERATION_TYPE = cmd.getOptionValue(GENERATION_TYPE_OPTION);
        
        UPDATE_TYPE = cmd.getOptionValue(UPDATE_TYPE_OPTION);

        // Look for optional args.
        if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {
            if (cmd.hasOption(ENTITY_ID_OPTION)) {
                ENTITY_ID = cmd.getOptionValue(ENTITY_ID_OPTION);
            } else {
                HelpFormatter formatter = new HelpFormatter();
                formatter.printHelp(COMMAND_LINE, options);
                System.exit(1);
            }
        }
        
        if (GENERATION_TYPE_INCREMENTAL.equals(GENERATION_TYPE))
                if (cmd.hasOption(TIMESTAMP_OPTION)) {
                        SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
                        try {
                            timeStamp = df.parse(cmd.getOptionValue(TIMESTAMP_OPTION));
                        } catch(Exception e) {
                                HelpFormatter formatter = new HelpFormatter();
                    formatter.printHelp(COMMAND_LINE, options);
                    System.exit(1);
                        }
                }
    }

    /**
     * Call method based on arguments
     * 
     * @throws Exception
     */
    private void callMethod() {
        boolean isSuccess = false;
        String logfile = "/tmp/content-from-cms.log";
        if (UPDATE_TYPE.equals(UPDATE_TYPE_CONTENT)) {
            logfile = "/tmp/content-from-cms.log";
            try {
                this.generateContent();
                isSuccess = true;
            } catch (Exception e) {
                log.error("Error generating content", e);
            }
        }

        if (UPDATE_TYPE.equals(UPDATE_TYPE_CATALOG)) {
            logfile = "/tmp/content-from-catalog.log";
            try {
                this.updatePrices();
                isSuccess = true;
            } catch (Exception e) {
                log.error("Error updating prices", e);
            }
        }

        GmailUtils gm = new GmailUtils();
        String[] sendTo = { "rajveer.singh@shop2020.in", "mandeep.dhir@shop2020.in", "pankaj.kankar@shop2020.in", "anupam.singh@shop2020.in", "amit.gupta@shop2020.in" };

        try {
            gm.sendSSLMessage(sendTo, "Content Generation Successful ? : "
                    + isSuccess, "Content generation completed at time : "
                    + Calendar.getInstance().getTime().toString(),
                    "build@shop2020.in", "cafe@nes", logfile);
        } catch (MessagingException e) {
            log.error("Could not send status mail", e);
        }
    }

    public boolean cleanDir(File dir, boolean deleteSelf) {
        if (dir.isDirectory()) {
            String[] children = dir.list();
            for (int i = 0; i < children.length; i++) {
                boolean success = cleanDir(new File(dir, children[i]), true);
                if (!success) {
                    return false;
                }
            }
        }

        // The directory is now empty so delete it
        if (deleteSelf) {
            return dir.delete();
        }

        return true;
    }

    private void removeOldResources() throws IOException {
        File f = new File(Utils.EXPORT_SOLR_PATH);
        if (f.exists()) {
            cleanDir(f, false);
        }
        
        for(String domainPath : DOMAINPATHS){
                String pathName = domainPath.split("\\.")[0].split(":")[0];
                File f1 = new File(Utils.EXPORT_PATH + "html/entities-" +  pathName);
                if (f1.exists()) {
                        cleanDir(f1, false);
                }else {
                        f1.mkdir();
                }
        }
    }

    /**
     * Update the prices in the generated content
     * 
     * @throws Exception
     */
    private void updatePrices() throws Exception {
        lastGenerationTime = new Long(0);
        if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {
            items = client.getItemsByCatalogId(Long.parseLong(ENTITY_ID));
            Iterator<Item> it = items.iterator();
            while(it.hasNext()){
                status st = it.next().getItemStatus();
                if(!(st.equals(status.ACTIVE) || st.equals(status.PAUSED) || 
                                        st.equals(status.COMING_SOON) || st.equals(status.PHASED_OUT))){
                        it.remove();
                }
            }
            try {
                //Generate prices and availability data for amazon
                AmazonSCDataGenerator.generatePricesAndAvailability(items);
            } catch (Exception e) {
                log.info("Could not generate Amazon prices and availability", e);
            }
            //ProductListGenerator.updatePriceForEntity(Long.parseLong(ENTITY_ID), items.get(0).getSellingPrice(), items.get(0).getMrp());
        } else {
                log.info("Before getting active items.");
            items = client.getAllItemsByStatus(status.ACTIVE);
            log.info("Before getting coming items.");
            items.addAll(client.getAllItemsByStatus(status.COMING_SOON));
            log.info("Before getting paused items.");
            items.addAll(client.getAllItemsByStatus(status.PAUSED));
            // Clean up the data from the solr directories.
            log.info("Before removing old resources.");
            removeOldResources();

        }

        // this still needs to be evolved. Must not be used.
        if (GENERATION_TYPE.equals(GENERATION_TYPE_INCREMENTAL)) {
        }

        // Populate the entityIdIemMap
        populateEntityIdItemMap();
        List<Long> inStockEntities = null;
        // Generate partners and json objects for phones only
        if (!GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {
                ProductListGenerator generator = new ProductListGenerator(entityIdItemMap);
                log.info("Before thinkdigit feed.");
                generator.generateThinkDigitFeed();
                inStockEntities = generator.getInStockCatalogItemIds();
                log.info("Before product list js.");
                generator.generateProductListJavascript();
                
                try     {
                        log.info("Before product list xml.");
                generator.generateProductsListXML();
                log.info("Before product accessories xml.");
                generator.generateAccessoriesXML();
                log.info("Before product camera xml.");
                generator.generateCamerasXML();
                log.info("Before product display ads.");
                        //generator.generateProductXMLForDisplayAds();
                } catch (Exception e) {
                        e.printStackTrace();
                }
                
        }
        PriceInsertor priceInserter = new PriceInsertor();

        Map<Long,List<String>> entityTags = client.getAllEntityTags();
        Map<Long, Integer> popularityMap = getPolularityMap();
        for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {
            long entityId = entry.getKey();
            List<Item> items = entry.getValue();
                    // TODO Domain name and destination directory should be read from
                    // properties file
            double minPrice = 0d;
            String availability = "In Stock";
            if (inStockEntities != null ) {
                if (!inStockEntities.contains(entry.getKey())) {
                        availability = "Out of Stock";
                }
            } else {
                availability = getAvaialability(entry.getValue());
            }
            StringBuilder priceString = new StringBuilder();
            StringBuilder availabilityString = new StringBuilder();
            boolean domainOnce = true;
            boolean sourceOnce = true;
            for(String domainPath : DOMAINPATHS){
                String domainName = domainPath;
                String pathName = domainPath.split("\\.")[0].split(":")[0];
                minPrice = priceInserter.insertPriceInHtml(items, entityId,
                                domainName, Utils.EXPORT_PATH + "html/entities-" +  pathName + "/", null);
                if(domainOnce){
                        priceString.append("<field name=\"F_50002\">" + minPrice + "</field>");
                        availabilityString.append("<field name=\"F_50028\">" + availability + "</field>");
                        if(entityTags.containsKey(entityId)) {
                                List<String> tags = entityTags.get(entityId);
                                for(String tag: tags){
                                        availabilityString.append("\n<field name=\"F_50029\">" + tag + "</field>");
                                }
                        }
                        if(popularityMap.containsKey(entityId)) {
                                availabilityString.append("\n<field name=\"F_50030\">" + popularityMap.get(entityId) + "</field>");
                        }else {
                                availabilityString.append("\n<field name=\"F_50030\">" + "0" + "</field>");
                        }
                        domainOnce = false;
                }
                if(sources != null){
                        for (Source source : sources) {
                        minPrice = priceInserter.insertPriceInHtml(items, entityId,
                                domainName, Utils.EXPORT_PATH + "html/entities-" + pathName + "/",
                                source);
                        if(sourceOnce){
                                priceString.append("<field name=\"F_50002_"
                                + source.getId() + "\">" + minPrice + "</field>");
                        }
                        }
                        sourceOnce = false;
                }
            }

            priceInserter.insertPriceInSolrData(entityId,
                    priceString.toString(), availabilityString.toString());
        }

        priceInserter.copySolrSchemaFiles();
        synonymTitlesExporter();
    }

    private Map<Long, Integer> getPolularityMap() {
                try {
                        Reader reader = new FileReader(Utils.EXPORT_PATH + Utils.POPULARITY_JSON);
                        return new Gson().fromJson(reader, new TypeToken<Map<Long, Integer>>() {}.getType());
                } catch (FileNotFoundException e) {
                        log.error("Could not read popularity file");
                        e.printStackTrace();
                        return new HashMap<Long, Integer>();
                }
        
        }

        private String getAvaialability(List<Item> value) {
        boolean isActive = true;
        try {
                        Client catalogClientProd = new CatalogClient(ConfigClientKeys.catalog_service_server_host_prod.toString(), ConfigClientKeys.catalog_service_server_port.toString()).getClient();
                        for (Item item : value ) {
                                if (item.getItemStatus().equals(status.ACTIVE)) {
                                        if(item.isRisky()){
                                        try {
                                                ItemShippingInfo isi = catalogClientProd.isActive(item.getId());
                                                isActive = isi.isIsActive();
                                        } catch (Exception e) {
                                                e.printStackTrace();
                                                isActive = true;
                                        }
                                }
                                }
                                if(isActive) break;
                        }
                } catch (TTransportException e) {
                        // TODO Auto-generated catch block
                        e.printStackTrace();
                }
                return isActive ? "In Stock" : "Out of Stock";
        }

        /**
     * Generates content for the specified entity embedding links to the
     * specified domain name.
     * 
     * The method will not generate content if one of the following conditions
     * is met:
     * <ol>
     * <li>The entity is not ready.
     * <li>The category has not been updated yet. (Set to -1).
     * <li>The content has not been updated.
     * </ol>
     * 
     * @throws
     */
    private void generateContent() throws Exception {
        if (GENERATION_TYPE.equals(GENERATION_TYPE_ALL)) {
            entities = CreationUtils.getEntities();
            lastGenerationTime = new Long(0);
        } else if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {
            entities = new HashMap<Long, Entity>();
            entities.put(Long.parseLong(ENTITY_ID),
                    CreationUtils.getEntity(Long.parseLong(ENTITY_ID)));
            lastGenerationTime = new Long(0);
        } else {
                entities = CreationUtils.getEntities();
            //  When we read lastGenerationTime from database
            //  then only we should mark the 
            //  current time as newLastGenerationTime
            if(timeStamp == null) {
                newLastGenerationTime = new Date().getTime();
                lastGenerationTime = CreationUtils.getLastContentGenerationTime();
            } else {
                lastGenerationTime = timeStamp.getTime();
            }
            
            log.info("lastGenerationTime: " + lastGenerationTime);
            if (lastGenerationTime == null) {
                lastGenerationTime = new Long(0);
            }
        } 

        // Filter invalid entities here
        List<Entity> validEntities = new ArrayList<Entity>();
        for (long entityID : entities.keySet()) {
            if (isValidEntity(entities.get(entityID))) {
                validEntities.add(entities.get(entityID));
            }
        }

        // Calculate comparison scores
        log.info("Calculating comparison scores");
        NewCMP cmp = new NewCMP(validEntities);
        Map<Long, Map<Long, Double>> slideScoresByEntity = cmp.getSlideScores();
        CreationUtils.storeSlideScores(slideScoresByEntity);

        // Fetch comparison statistics everyday and store them in BDB
        log.info("Fetching comparison statistics");
        ComparisonStatsFetcher csf = new ComparisonStatsFetcher();
        csf.fetchAndStoreComparisonStats();

        // Upload catalog to Google App Engine.
        if (GENERATION_TYPE.equals(GENERATION_TYPE_ALL)) {
            log.info("Uploading Catalog to Google app engine");
            List<Item> allItems = client.getAllItems(false);
            allItems.addAll(client.getAllItems(true));
            CatalogUploderToGAE catalogUploaderToGAE = new CatalogUploderToGAE();
            catalogUploaderToGAE.uploadItems(allItems);
        }

        items = client.getAllItemsByStatus(status.ACTIVE);
        items.addAll(client.getAllItemsByStatus(status.PAUSED));
        items.addAll(client.getAllItemsByStatus(status.CONTENT_COMPLETE));
        items.addAll(client.getAllItemsByStatus(status.COMING_SOON));
        populateEntityIdItemMap();

        //FIXME Avoiding the finding of accesories, as list of categories for which we need to find accessories is hardocoded in code. 
        // We need to make that configurable. Also creating ticket to improve it.
        try{
                log.info("Finding accessories");
                AccessoriesFinder af = new AccessoriesFinder(new HashSet<Long>(allValidEntityIds));
                Map<Long, Map<Long, List<Long>>> relatedAccessories = af.findAccessories();
                CreationUtils.storeRelatedAccessories(relatedAccessories);
        }catch (Exception e) {
                log.error("Error while generating accessories" + e);
                }
        
        log.info("Writing JSON file for special pages");
        SpecialPageJSONConvertor bjc = new SpecialPageJSONConvertor();
        bjc.writeToJSONFile(new File(Utils.EXPORT_JAVASCRIPT_CONTENT_PATH
                + "special-pages.json"));

        log.info("Generating velocity templates, images, documents etc.");
        NewVUI vui = new NewVUI(lastGenerationTime);
        for (Entity entity : validEntities) {
            log.info("Processing Entityid: " + entity.getID());
            vui.generateContentForOneEntity(entity, Utils.EXPORT_VELOCITY_PATH);
        }

        // Generate synonyms list. This will be used in PriceComparisonTool to
        // resolve the product names.
        log.info("Generating synonyms");
        SynonymExporter sx = new SynonymExporter();
        sx.storeSynonyms(validEntities);
        
        List<Entity> allValidEntities;
        
        if (GENERATION_TYPE.equals(GENERATION_TYPE_ALL))        {
                allValidEntities = validEntities;
            
        } else  {
                allValidEntities = new ArrayList<Entity>();
                        for (Long entityId : allValidEntityIds) {
                                allValidEntities.add(CreationUtils.getEntity(entityId));
                        }
        }
        
        log.info("Generating HTML for Site Index");
        ProductIndexGenerator indexGenerator = new ProductIndexGenerator(allValidEntities);
        indexGenerator.generate();

        log.info("Generating HTML for Site for Product Documents");
        ProductDocumentsGenerator asGenerator = new ProductDocumentsGenerator(allValidEntities);
        asGenerator.generate();
        
        log.info("Generating HTML for Accessories Compatibility Index");
        CompatibleAccessoriesIndexGenerator generator = new CompatibleAccessoriesIndexGenerator(allValidEntities);
        generator.generate();

        log.info("Generating HTML for Most Frequently searched keywords");
        MostFrequentlySearchedKeywords mfsk = new MostFrequentlySearchedKeywords();
        mfsk.generate();
        
        log.info("Generating HTML for Most Compared Index");
        MostComparedIndexGenerator mostCompGenerator = new MostComparedIndexGenerator(allValidEntityIds);
        mostCompGenerator.generate();

        log.info("Generating XML for Mobile Site XML feed");
        MobileSiteDataXMLGenerator mSiteXMLGenerator = new MobileSiteDataXMLGenerator(allValidEntities);
        mSiteXMLGenerator.generate();
        
        if (newLastGenerationTime != 0) {
            CreationUtils.storeLastContentGenerationTime(newLastGenerationTime);
        }


        log.info("Generating Solr files");
        NewIR ir = new NewIR(validEntities);
        ir.exportIRData();
        // ir.transformIrDataXMLtoSolrXML();
        ir.exportIRMetaData();
        ir.transformIrMetaDataXMLtoSolrSchemaXML();
        ir.transformIrMetaDataXMLtoSolrCatchAllXML();

        for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {
            List<Item> items = entry.getValue();
            for (Item item : items) {
                if (item.getItemStatus() == status.CONTENT_COMPLETE || item.getItemStatus() == status.COMING_SOON) {
                        if(item.getStartDate() <= new Date().getTime() + ONE_DAY){
                                item.setItemStatus(status.ACTIVE);
                                item.setStatus_description("This item is active");
                        } else {
                                item.setItemStatus(status.COMING_SOON);
                                String productName = getProductName(item);
                                String statusDescription = productName + " is coming soon.";
                                if(item.getExpectedArrivalDate()>new Date().getTime() + ONE_DAY){
                                        statusDescription = productName + " will be available by " 
                                        + new SimpleDateFormat("dd/MM/yy").format(new Date(item.getExpectedArrivalDate()));
                                }
                                //Send alert to Category team one day before expected arrival date
                                //So they may change the expected arrival date if they want to.
                                if(item.getExpectedArrivalDate() < new Date().getTime() + 2*ONE_DAY && 
                                                item.getExpectedArrivalDate() > new Date().getTime() + ONE_DAY) {
                                                alertItems.add(item);
                                }
                                item.setStatus_description(statusDescription);
                        }
                    client.updateItem(item);
                }
            }
        }
        sendAlertToCategoryTeam(alertItems);
        try {
            //generate products list that is to be uploaded in Amazon.
                AmazonSCDataGenerator ascdGenerator = new AmazonSCDataGenerator(validEntities, GENERATION_TYPE);
            ascdGenerator.generateSCProdData();
        } catch (Exception e) {
                e.printStackTrace();
            log.info("Could not generate Amazon data", e);
        }
    }

    private void sendAlertToCategoryTeam(List<Item> items) {
        if(items!=null && items.size()!=0){
                        GmailUtils util = new GmailUtils();
                        String[] recipients = {"amit.gupta@shop2020.in", "chaitnaya.vats@shop2020.in", "khushal.bhatia@shop2020.in"};
                        String from = "build@shop2020.in";
                        String password = "cafe@nes";
                        String subject = Utils.EXPECTED_ARRIVAL_ACHIEVED_TEMPLATE;
                        StringBuffer message = new StringBuffer("Please check the following items:\n");
                        List<File> emptyList = new ArrayList<File>();
                        for( Item item : items){
                                message.append("\t" + getProductName(item));
                        }
                        try {
                                util.sendSSLMessage(recipients, subject, message.toString(), from, password, emptyList);
                        } catch (Exception e){
                                log.info("Could not send alert" + e);
                        }
        }
        }

        private String getProductName(Item item) {
        String brand = item.getBrand();
                String modelName = item.getModelName();
                String modelNumber = item.getModelNumber();
                String product = "";
                if(StringUtils.isEmpty(modelName)){
                        product = brand + " " + modelNumber;
                }else {
                        product = brand + " " + modelName + " " + modelNumber;
                }
                return product;
        }

    /**
     * Checks weather entity is valid or not. Entity will be invalid in one of
     * these cases:
     * <ol>
     * <li>The entity is not ready.
     * <li>The category has not been updated yet. (Set to -1).
     * <li>Content has not been updated after last content generation timestamp.
     * </ol>
     * 
     * @param entity
     * @return
     * @throws Exception
     */
    private boolean isValidEntity(Entity entity) throws Exception {
        ExpandedEntity expEntity = new ExpandedEntity(entity);
        EntityState state = CreationUtils.getEntityState(entity.getID());
        long categoryID = expEntity.getCategoryID();

        if (state.getStatus() != EntityStatus.READY || categoryID == -1) {
            return false;
        }
        if (state.getMerkedReadyOn().getTime() < this.lastGenerationTime) {
            return false;
        }
        return true;
    }

    private void populateEntityIdItemMap() {
        Date todate = new Date();
        Utils.info("Processing " + items.size() + " items");
        for (Item item : items) {
            Utils.info(item.getId() + ":" + item.getItemStatus() + ":" + item.getCatalogItemId());
            // TODO Can be removed as we are checking in calling function
            if (!(item.getItemStatus() == status.ACTIVE
                    || item.getItemStatus() == status.CONTENT_COMPLETE || item
                    .getItemStatus() == status.PAUSED || item.getItemStatus() == status.COMING_SOON)) {
                continue;
            }
            Utils.info(item.getStartDate() + ":" + item.getSellingPrice());
            
                        if (todate.getTime() < item.getStartDate()
                                        && (!item.isSetExpectedArrivalDate() || todate.getTime() < item.getComingSoonStartDate()) 
                                        || item.getSellingPrice() == 0) {
                                continue;
                        }
            Utils.info(item.getId() + " Item is adding");
            List<Item> itemList = entityIdItemMap.get(item.getCatalogItemId());
            if (itemList == null) {
                itemList = new ArrayList<Item>();
            } 
            itemList.add(item);
            entityIdItemMap.put(item.getCatalogItemId(), itemList);
        }

        Utils.info("Processing " + entityIdItemMap.size() + " entities");
        // Remove all items which have not been updated since last content
        // generation.
        List<Long> removeEntities = new ArrayList<Long>();
        for (Long entityId : entityIdItemMap.keySet()) {
            boolean isValidEntity = false;
            // If any one of the items has been updated before current
            // timestamp, than we generate content for whole entity
            for (Item item : entityIdItemMap.get(entityId)) {
                if (item.getUpdatedOn() > lastGenerationTime || item.getItemStatus()==status.COMING_SOON) {
                    isValidEntity = true;
                }
            }
            if (!isValidEntity) {
                removeEntities.add(entityId);
            }
        }
        //Simply assign allValidEntityIds to a class variable as these need to be used where all valid entites
        //are needed.
        allValidEntityIds = new ArrayList<Long>(entityIdItemMap.keySet());
        for (Long entityId : removeEntities) {
            entityIdItemMap.remove(entityId);
        }

        Utils.info("Final valid entities to be processed: " + entityIdItemMap.size());
    }

    private void synonymTitlesExporter() {
        SynonymExporter sx = new SynonymExporter();
        Map<Long, Map<String,List<String>>> synonyms = sx.getSynonyms();
        Map<String, List<String>> finalsynonyms = new HashMap<String, List<String>>();
        for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {
            long entityId = entry.getKey();
                try{
                    String brand = "";
                    String originalModelName = "";
                    String originalModelNumber = "";
                    List<String> modelNameSynonyms =  new ArrayList<String>();
                    List<String> modelNumberSynonyms =  new ArrayList<String>();
                    List<String> titles = new ArrayList<String>();
                    Map<String,List<String>> synonymMap = synonyms.get(entityId);
                    if(synonymMap != null && !synonymMap.isEmpty()){
                        if(synonymMap.get("ORIGINAL_MODEL_NAME") != null && !synonymMap.get("ORIGINAL_MODEL_NAME").isEmpty()){
                                modelNameSynonyms.addAll(synonymMap.get("ORIGINAL_MODEL_NAME"));
                                originalModelName = synonymMap.get("ORIGINAL_MODEL_NAME").get(0);
                        }
                        if(synonymMap.get("MODEL_NAME") != null && !synonymMap.get("MODEL_NAME").isEmpty()){
                                modelNameSynonyms.addAll(synonymMap.get("MODEL_NAME"));
                        }
                        if(synonymMap.get("ORIGINAL_MODEL_NUMBER") != null && !synonymMap.get("ORIGINAL_MODEL_NUMBER").isEmpty()){
                                modelNumberSynonyms.addAll(synonymMap.get("ORIGINAL_MODEL_NUMBER"));
                                originalModelNumber = synonymMap.get("ORIGINAL_MODEL_NUMBER").get(0);
                        }
                        if(synonymMap.get("MODEL_NUMBER") != null && !synonymMap.get("MODEL_NUMBER").isEmpty()){
                                modelNumberSynonyms.addAll(synonymMap.get("MODEL_NUMBER"));
                        }
                        brand = ((synonymMap.get("ORIGINAL_BRAND") != null && !synonymMap.get("ORIGINAL_BRAND").isEmpty()) ? synonymMap.get("ORIGINAL_BRAND").get(0) : "");
                    }
                    for(String model_name: modelNameSynonyms){
                        for(String model_number: modelNumberSynonyms){
                                String title = brand + " " + model_name + " " + model_number;
                                title = title.replaceAll("  ", " ");
                                titles.add(title);
                        }
                    }
                    String originaltitle = brand + " " + originalModelName + " " + originalModelNumber;
                    originaltitle = originaltitle.replaceAll("  ", " ");
                    originaltitle = originaltitle.trim();
                    if(!originaltitle.isEmpty()) {
                        finalsynonyms.put(originaltitle, titles);
                    }
                } catch (Exception e) {
                                e.printStackTrace();
                        }
        }
        
        String autosuggestFilename = Utils.EXPORT_JAVASCRIPT_CONTENT_PATH + "autosuggest.json";
        Gson gson = new Gson();
                try {
                        DBUtils.store(gson.toJson(finalsynonyms), autosuggestFilename);
                } catch (Exception e) {
                        e.printStackTrace();
                }
    }
}