Subversion Repositories SmartDukaan

Rev

Rev 5106 | Rev 5227 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | RSS feed 

package in.shop2020.util;

import in.shop2020.metamodel.core.Entity;
import in.shop2020.metamodel.core.EntityState;
import in.shop2020.metamodel.core.EntityStatus;
import in.shop2020.metamodel.util.CreationUtils;
import in.shop2020.metamodel.util.ExpandedEntity;
import in.shop2020.model.v1.catalog.InventoryService.Client;
import in.shop2020.model.v1.catalog.Item;
import in.shop2020.model.v1.catalog.Source;
import in.shop2020.model.v1.catalog.status;
import in.shop2020.thrift.clients.CatalogClient;
import in.shop2020.ui.util.CatalogUploderToGAE;
import in.shop2020.ui.util.ComparisonStatsFetcher;
import in.shop2020.ui.util.NewVUI;
import in.shop2020.ui.util.PriceInsertor;
import in.shop2020.ui.util.SpecialPageJSONConvertor;
import in.shop2020.utils.GmailUtils;

import java.io.File;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

import javax.mail.MessagingException;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import com.google.gson.Gson;

public class ContentGenerationUtility {
    private static final String   UPDATE_TYPE_CATALOG         = "CATALOG";

    private static final String   UPDATE_TYPE_CONTENT         = "CONTENT";

    private static final String   COMMAND_LINE                = "java ContentGenerationUtility.class -t { ALL | INCREMENTAL | ONE } -s { yyyy-MM-dd-HH-mm-ss } -u { CONTENT | CATALOG } -e {EntityId} ";

    private static Log            log                         = LogFactory
                                                                      .getLog(ContentGenerationUtility.class);

    // Commandline options
    private static Options        options                     = null;
    private static final String   GENERATION_TYPE_INCREMENTAL = "INCREMENTAL";
    private static final String   GENERATION_TYPE_ALL         = "ALL";
    private static final String   GENERATION_TYPE_ONE         = "ONE";
    private static final String   UPDATE_TYPE_OPTION          = "u";
    private static final String   GENERATION_TYPE_OPTION      = "t";
    private static final String   ENTITY_ID_OPTION            = "e";
    private static final String   TIMESTAMP_OPTION            = "s";

    // Default values of cmdline options
    private static String         UPDATE_TYPE                 = UPDATE_TYPE_CONTENT;
    private static String         GENERATION_TYPE             = GENERATION_TYPE_INCREMENTAL;
    private static String         ENTITY_ID                   = "ALL";

    private Date                                  timeStamp                                       = null;
    private CommandLine           cmd                         = null;
    private Map<Long, List<Item>> entityIdItemMap             = new LinkedHashMap<Long, List<Item>>();
    private Long                  lastGenerationTime          = 0l;
    private Map<Long, Entity>     entities;
    private List<Item>            items;
    private List<Source>          sources;
    private CatalogClient         csc;
    private Client                client;

    private long                  newLastGenerationTime;

    static {
        options = new Options();
        options.addOption(GENERATION_TYPE_OPTION, true, "Generation type");
        options.addOption(UPDATE_TYPE_OPTION, true, "Default is : "
                + UPDATE_TYPE);
        options.addOption(ENTITY_ID_OPTION, true, "all entities " + ENTITY_ID
                + " by default");
        options.addOption(TIMESTAMP_OPTION, true, "Manual timestamp");

    }

    public ContentGenerationUtility() throws Exception {
        csc = new CatalogClient();
        client = csc.getClient();
        sources = client.getAllSources();
    }

    /**
     * @param args
     * @throws Exception
     */
    public static void main(String[] args) throws Exception {
        ContentGenerationUtility cgu = new ContentGenerationUtility();

        // Load arguments
        cgu.loadArgs(args);

        // Call method based on arguments
        cgu.callMethod();
    }

    /**
     * Validate and set command line arguments. Exit after printing usage if
     * anything is astray
     * 
     * @param args
     *            String[] args as featured in public static void main()
     */
    private void loadArgs(String[] args) {
        CommandLineParser parser = new PosixParser();

        try {
            cmd = parser.parse(options, args);
        } catch (ParseException e) {
            log.error("Error parsing arguments", e);
            System.exit(1);
        }

        // Check for mandatory args
        if (!(cmd.hasOption(GENERATION_TYPE_OPTION) && cmd
                .hasOption(UPDATE_TYPE_OPTION))) {
            HelpFormatter formatter = new HelpFormatter();
            formatter.printHelp(COMMAND_LINE, options);
            System.exit(1);
        }
        
        
        GENERATION_TYPE = cmd.getOptionValue(GENERATION_TYPE_OPTION);
        
        UPDATE_TYPE = cmd.getOptionValue(UPDATE_TYPE_OPTION);

        // Look for optional args.
        if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {
            if (cmd.hasOption(ENTITY_ID_OPTION)) {
                ENTITY_ID = cmd.getOptionValue(ENTITY_ID_OPTION);
            } else {
                HelpFormatter formatter = new HelpFormatter();
                formatter.printHelp(COMMAND_LINE, options);
                System.exit(1);
            }
        }
        
        if (GENERATION_TYPE_INCREMENTAL.equals(GENERATION_TYPE))
                if (cmd.hasOption(TIMESTAMP_OPTION)) {
                        SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
                        try {
                            timeStamp = df.parse(cmd.getOptionValue(TIMESTAMP_OPTION));
                        } catch(Exception e) {
                                HelpFormatter formatter = new HelpFormatter();
                    formatter.printHelp(COMMAND_LINE, options);
                    System.exit(1);
                        }
                }
    }

    /**
     * Call method based on arguments
     * 
     * @throws Exception
     */
    private void callMethod() {
        boolean isSuccess = false;
        String logfile = "/tmp/content-from-cms.log";
        if (UPDATE_TYPE.equals(UPDATE_TYPE_CONTENT)) {
            logfile = "/tmp/content-from-cms.log";
            try {
                this.generateContent();
                isSuccess = true;
            } catch (Exception e) {
                log.error("Error generating content", e);
            }
        }

        if (UPDATE_TYPE.equals(UPDATE_TYPE_CATALOG)) {
            logfile = "/tmp/content-from-catalog.log";
            try {
                this.updatePrices();
                isSuccess = true;
            } catch (Exception e) {
                log.error("Error updating prices", e);
            }
        }

        GmailUtils gm = new GmailUtils();
        String[] sendTo = { "rajveer.singh@shop2020.in", "mandeep.dhir@shop2020.in", "pankaj.kankar@shop2020.in", "anupam.singh@shop2020.in", "amit.gupta@shop2020.in" };

        try {
            gm.sendSSLMessage(sendTo, "Content Generation Successful ? : "
                    + isSuccess, "Content generation completed at time : "
                    + Calendar.getInstance().getTime().toString(),
                    "build@shop2020.in", "cafe@nes", logfile);
        } catch (MessagingException e) {
            log.error("Could not send status mail", e);
        }
    }

    public boolean cleanDir(File dir, boolean deleteSelf) {
        if (dir.isDirectory()) {
            String[] children = dir.list();
            for (int i = 0; i < children.length; i++) {
                boolean success = cleanDir(new File(dir, children[i]), true);
                if (!success) {
                    return false;
                }
            }
        }

        // The directory is now empty so delete it
        if (deleteSelf) {
            return dir.delete();
        }

        return true;
    }

    private void removeOldResources() throws IOException {
        File f = new File(Utils.EXPORT_SOLR_PATH);
        if (f.exists()) {
            cleanDir(f, false);
        }

        File f1 = new File(Utils.EXPORT_ENTITIES_PATH_LOCALHOST);
        if (f1.exists()) {
            cleanDir(f1, false);
        }

        File f2 = new File(Utils.EXPORT_ENTITIES_PATH_SAHOLIC);
        if (f2.exists()) {
            cleanDir(f2, false);
        }

        File f3 = new File(Utils.EXPORT_ENTITIES_PATH_SHOP2020);
        if (f3.exists()) {
            cleanDir(f3, false);
        }
    }

    /**
     * Update the prices in the generated content
     * 
     * @throws Exception
     */
    private void updatePrices() throws Exception {
        lastGenerationTime = new Long(0);
        if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {
            items = client.getItemsByCatalogId(Long.parseLong(ENTITY_ID));
        } else {
            items = client.getAllItemsByStatus(status.ACTIVE);
            items.addAll(client.getAllItemsByStatus(status.PAUSED));
            try {
                //Generate prices and availability data for amazon
                AmazonSCDataGenerator.generatePricesAndAvailability();
            } catch (Exception e) {
                log.info("Could not generate Amazon prices and availability", e);
            }
            // Clean up the data from the solr directories.
            removeOldResources();

        }

        // this still needs to be evolved. Must not be used.
        if (GENERATION_TYPE.equals(GENERATION_TYPE_INCREMENTAL)) {
        }

        // Populate the entityIdIemMap
        populateEntityIdItemMap();

        PriceInsertor priceInserter = new PriceInsertor();

        for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {
            long entityId = entry.getKey();
            List<Item> items = entry.getValue();
            // TODO Domain name and destination directory should be read from
            // properties file
            double minPrice = priceInserter.insertPriceInHtml(items, entityId,
                    "saholic.com", Utils.EXPORT_ENTITIES_PATH_SAHOLIC, null);
            priceInserter.insertPriceInHtml(items, entityId, "shop2020.in",
                    Utils.EXPORT_ENTITIES_PATH_SHOP2020, null);
            priceInserter.insertPriceInHtml(items, entityId, "localhost:8090",
                    Utils.EXPORT_ENTITIES_PATH_LOCALHOST, null);
            StringBuilder priceString = new StringBuilder(
                    "<field name=\"F_50002\">" + minPrice + "</field>");

            if (sources != null) {
                for (Source source : sources) {
                    minPrice = priceInserter.insertPriceInHtml(items, entityId,
                            "saholic.com", Utils.EXPORT_ENTITIES_PATH_SAHOLIC,
                            source);
                    priceInserter.insertPriceInHtml(items, entityId,
                            "shop2020.in", Utils.EXPORT_ENTITIES_PATH_SHOP2020,
                            source);
                    priceInserter.insertPriceInHtml(items, entityId,
                            "localhost:8090",
                            Utils.EXPORT_ENTITIES_PATH_LOCALHOST, source);
                    priceString.append("<field name=\"F_50002_"
                            + source.getId() + "\">" + minPrice + "</field>");
                }
            }

            priceInserter.insertPriceInSolrData(entityId,
                    priceString.toString());
        }

        priceInserter.copySolrSchemaFiles();
        synonymTitlesExporter();
        // Generate partners and json objects for phones only
        if (!GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {
            ProductListGenerator generator = new ProductListGenerator(entityIdItemMap);
            generator.generateProductsListXML();
            generator.generateProductListJavascript();
            
            
            try {
                generator.generateProductXMLForDisplayAds();
            } catch (Exception e) {
                                
                        }
            
        }
    }

    /**
     * Generates content for the specified entity embedding links to the
     * specified domain name.
     * 
     * The method will not generate content if one of the following conditions
     * is met:
     * <ol>
     * <li>The entity is not ready.
     * <li>The category has not been updated yet. (Set to -1).
     * <li>The content has not been updated.
     * </ol>
     * 
     * @throws
     */
    private void generateContent() throws Exception {
        if (GENERATION_TYPE.equals(GENERATION_TYPE_ALL)) {
            entities = CreationUtils.getEntities();
            lastGenerationTime = new Long(0);
        } else if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {
            entities = new HashMap<Long, Entity>();
            entities.put(Long.parseLong(ENTITY_ID),
                    CreationUtils.getEntity(Long.parseLong(ENTITY_ID)));
            lastGenerationTime = new Long(0);
        } else {
                entities = CreationUtils.getEntities();
            //  When we read lastGenerationTime from database
            //  then only we should mark the 
            //  current time as newLastGenerationTime
            if(timeStamp == null) {
                newLastGenerationTime = new Date().getTime();
                lastGenerationTime = CreationUtils.getLastContentGenerationTime();
            } else {
                lastGenerationTime = timeStamp.getTime();
            }
            
            log.info("lastGenerationTime: " + lastGenerationTime);
            if (lastGenerationTime == null) {
                lastGenerationTime = new Long(0);
            }
        } 

        // Filter invalid entities here
        List<Entity> validEntities = new ArrayList<Entity>();
        for (long entityID : entities.keySet()) {
            if (isValidEntity(entities.get(entityID))) {
                validEntities.add(entities.get(entityID));
            }
        }

        // Calculate comparison scores
        log.info("Calculating comparison scores");
        NewCMP cmp = new NewCMP(validEntities);
        Map<Long, Map<Long, Double>> slideScoresByEntity = cmp.getSlideScores();
        CreationUtils.storeSlideScores(slideScoresByEntity);

        // Fetch comparison statistics everyday and store them in BDB
        log.info("Fetching comparison statistics");
        ComparisonStatsFetcher csf = new ComparisonStatsFetcher();
        csf.fetchAndStoreComparisonStats();

        // Upload catalog to Google App Engine.
        if (GENERATION_TYPE.equals(GENERATION_TYPE_ALL)) {
            log.info("Uploading Catalog to Google app engine");
            List<Item> allItems = client.getAllItems(false);
            allItems.addAll(client.getAllItems(true));
            CatalogUploderToGAE catalogUploaderToGAE = new CatalogUploderToGAE();
            catalogUploaderToGAE.uploadItems(allItems);
        }

        items = client.getAllItemsByStatus(status.ACTIVE);
        items.addAll(client.getAllItemsByStatus(status.PAUSED));
        items.addAll(client.getAllItemsByStatus(status.CONTENT_COMPLETE));
        populateEntityIdItemMap();

        //FIXME Avoiding the finding of accesories, as list of categories for which we need to find accessories is hardocoded in code. 
        // We need to make that configurable. Also creating ticket to improve it.
        try{
                log.info("Finding accessories");
                AccessoriesFinder af = new AccessoriesFinder(entityIdItemMap.keySet());
                Map<Long, Map<Long, List<Long>>> relatedAccessories = af.findAccessories();
                CreationUtils.storeRelatedAccessories(relatedAccessories);
        }catch (Exception e) {
                log.error("Error while generating accessories" + e);
                }
        
        log.info("Writing JSON file for special pages");
        SpecialPageJSONConvertor bjc = new SpecialPageJSONConvertor();
        bjc.writeToJSONFile(new File(Utils.EXPORT_JAVASCRIPT_CONTENT_PATH
                + "special-pages.json"));

        log.info("Generating velocity templates, images, documents etc.");
        NewVUI vui = new NewVUI(lastGenerationTime);
        for (Entity entity : validEntities) {
            log.info("Processing Entityid: " + entity.getID());
            vui.generateContentForOneEntity(entity, Utils.EXPORT_VELOCITY_PATH);
        }

        // Generate synonyms list. This will be used in PriceComparisonTool to
        // resolve the product names.
        log.info("Generating synonyms");
        SynonymExporter sx = new SynonymExporter();
        sx.storeSynonyms(validEntities);
        
        // Generate HTML for Site Index
        log.info("Generating HTML for Site Index");
        ProductIndexGenerator indexGenerator = new ProductIndexGenerator(validEntities);
        indexGenerator.generate();
        
        log.info("Generating HTML for Accessories Compatibility Index");
        CompatibleAccessoriesIndexGenerator generator = new CompatibleAccessoriesIndexGenerator();
        generator.generate();
        
        if (newLastGenerationTime != 0) {
            CreationUtils.storeLastContentGenerationTime(newLastGenerationTime);
        }


        log.info("Generating Solr files");
        NewIR ir = new NewIR(validEntities);
        ir.exportIRData();
        // ir.transformIrDataXMLtoSolrXML();
        ir.exportIRMetaData();
        ir.transformIrMetaDataXMLtoSolrSchemaXML();
        ir.transformIrMetaDataXMLtoSolrCatchAllXML();

        for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {
            List<Item> items = entry.getValue();
            for (Item item : items) {
                if (item.getItemStatus() == status.CONTENT_COMPLETE) {
                    item.setItemStatus(status.ACTIVE);
                    item.setStatus_description("This item is active");
                    client.updateItem(item);
                }
            }
        }
        try {
            //generate products list that is to be uploaded in Amazon.
            AmazonSCDataGenerator.generateSCProdData(validEntities, GENERATION_TYPE);
        } catch (Exception e) {
                e.printStackTrace();
            log.info("Could not generate Amazon data", e);
        }
    }

    /**
     * Checks weather entity is valid or not. Entity will be invalid in one of
     * these cases:
     * <ol>
     * <li>The entity is not ready.
     * <li>The category has not been updated yet. (Set to -1).
     * <li>Content has not been updated after last content generation timestamp.
     * </ol>
     * 
     * @param entity
     * @return
     * @throws Exception
     */
    private boolean isValidEntity(Entity entity) throws Exception {
        ExpandedEntity expEntity = new ExpandedEntity(entity);
        EntityState state = CreationUtils.getEntityState(entity.getID());
        long categoryID = expEntity.getCategoryID();

        if (state.getStatus() != EntityStatus.READY || categoryID == -1) {
            return false;
        }
        if (state.getMerkedReadyOn().getTime() < this.lastGenerationTime) {
            return false;
        }
        return true;
    }

    private void populateEntityIdItemMap() {
        Date todate = new Date();
        Utils.info("Processing " + items.size() + " items");
        for (Item item : items) {
            Utils.info(item.getId() + ":" + item.getItemStatus() + ":" + item.getCatalogItemId());
            // TODO Can be removed as we are checking in calling function
            if (!(item.getItemStatus() == status.ACTIVE
                    || item.getItemStatus() == status.CONTENT_COMPLETE || item
                    .getItemStatus() == status.PAUSED)) {
                continue;
            }
            Utils.info(item.getStartDate() + ":" + item.getSellingPrice());
            if (todate.getTime() < item.getStartDate()
                    || item.getSellingPrice() == 0) {
                continue;
            }
            Utils.info(item.getId() + " Item is adding");
            List<Item> itemList = entityIdItemMap.get(item.getCatalogItemId());
            if (itemList == null) {
                itemList = new ArrayList<Item>();
            }
            itemList.add(item);
            entityIdItemMap.put(item.getCatalogItemId(), itemList);
        }

        Utils.info("Processing " + entityIdItemMap.size() + " entities");
        // Remove all items which have not been updated since last content
        // generation.
        List<Long> removeEntities = new ArrayList<Long>();
        for (Long entityId : entityIdItemMap.keySet()) {
            boolean isValidEntity = false;
            // If any one of the items has been updated before current
            // timestamp, than we generate content for whole entity
            for (Item item : entityIdItemMap.get(entityId)) {
                if (item.getUpdatedOn() > lastGenerationTime) {
                    isValidEntity = true;
                }
            }
            if (!isValidEntity) {
                removeEntities.add(entityId);
            }
        }
        for (Long entityId : removeEntities) {
            entityIdItemMap.remove(entityId);
        }

        Utils.info("Final valid entities to be processed: " + entityIdItemMap.size());
    }

    private void synonymTitlesExporter() {
        SynonymExporter sx = new SynonymExporter();
        Map<Long, Map<String,List<String>>> synonyms = sx.getSynonyms();
        Map<Long, List<String>> finalsynonyms = new HashMap<Long, List<String>>();
        for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {
            long entityId = entry.getKey();
                try{
                    String brand = "";
                    List<String> modelNameSynonyms =  new ArrayList<String>();
                    List<String> modelNumberSynonyms =  new ArrayList<String>();
                    List<String> titles = new ArrayList<String>();
                    Map<String,List<String>> synonymMap = synonyms.get(entityId);
                    if(synonymMap != null && !synonymMap.isEmpty()){
                        if(synonymMap.get("ORIGINAL_MODEL_NAME") != null && !synonymMap.get("ORIGINAL_MODEL_NAME").isEmpty()){
                                modelNameSynonyms.addAll(synonymMap.get("ORIGINAL_MODEL_NAME"));
                        }
                        if(synonymMap.get("MODEL_NAME") != null && !synonymMap.get("MODEL_NAME").isEmpty()){
                                modelNameSynonyms.addAll(synonymMap.get("MODEL_NAME"));
                        }
                        if(synonymMap.get("ORIGINAL_MODEL_NUMBER") != null && !synonymMap.get("ORIGINAL_MODEL_NUMBER").isEmpty()){
                                modelNumberSynonyms.addAll(synonymMap.get("ORIGINAL_MODEL_NUMBER"));
                        }
                        if(synonymMap.get("MODEL_NUMBER") != null && !synonymMap.get("MODEL_NUMBER").isEmpty()){
                                modelNumberSynonyms.addAll(synonymMap.get("MODEL_NUMBER"));
                        }
                        brand = ((synonymMap.get("ORIGINAL_BRAND") != null && !synonymMap.get("ORIGINAL_BRAND").isEmpty()) ? synonymMap.get("ORIGINAL_BRAND").get(0) : "");
                    }
                    for(String model_name: modelNameSynonyms){
                        for(String model_number: modelNumberSynonyms){
                                String title = brand + " " + model_name + " " + model_number;
                                title = title.replaceAll("  ", " ");
                                titles.add(title);
                        }
                    }
                    finalsynonyms.put(entityId, titles);
                } catch (Exception e) {
                                e.printStackTrace();
                        }
        }
        
        String autosuggestFilename = Utils.EXPORT_JAVASCRIPT_CONTENT_PATH + "autosuggest.json";
        Gson gson = new Gson();
                try {
                        DBUtils.store(gson.toJson(finalsynonyms), autosuggestFilename);
                } catch (Exception e) {
                        e.printStackTrace();
                }
    }
}