Subversion Repositories SmartDukaan

package in.shop2020.util;

import in.shop2020.metamodel.core.Entity;

import in.shop2020.metamodel.core.EntityState;

import in.shop2020.metamodel.core.EntityStatus;

import in.shop2020.metamodel.util.CreationUtils;

import in.shop2020.metamodel.util.ExpandedEntity;

import in.shop2020.model.v1.catalog.InventoryService.Client;

import in.shop2020.model.v1.catalog.Item;

import in.shop2020.model.v1.catalog.Source;

import in.shop2020.model.v1.catalog.status;

import in.shop2020.thrift.clients.CatalogClient;

import in.shop2020.ui.util.CatalogUploderToGAE;

import in.shop2020.ui.util.ComparisonStatsFetcher;

import in.shop2020.ui.util.NewVUI;

import in.shop2020.ui.util.PriceInsertor;

import in.shop2020.ui.util.SpecialPageJSONConvertor;

import in.shop2020.utils.GmailUtils;

import java.io.File;

import java.io.IOException;

import java.text.SimpleDateFormat;

import java.util.ArrayList;

import java.util.Calendar;

import java.util.Date;

import java.util.HashMap;

import java.util.LinkedHashMap;

import java.util.List;

import java.util.Map;

import javax.mail.MessagingException;

import org.apache.commons.cli.CommandLine;

import org.apache.commons.cli.CommandLineParser;

import org.apache.commons.cli.HelpFormatter;

import org.apache.commons.cli.Options;

import org.apache.commons.cli.ParseException;

import org.apache.commons.cli.PosixParser;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.google.gson.Gson;

public class ContentGenerationUtility {

    private static final String   UPDATE_TYPE_CATALOG         = "CATALOG";

    private static final String   UPDATE_TYPE_CONTENT         = "CONTENT";

    private static final String   COMMAND_LINE                = "java ContentGenerationUtility.class -t { ALL | INCREMENTAL | ONE } -s { yyyy-MM-dd-HH-mm-ss } -u { CONTENT | CATALOG } -e {EntityId} ";

    private static Log            log                         = LogFactory

                                                                      .getLog(ContentGenerationUtility.class);

    // Commandline options

    private static Options        options                     = null;

    private static final String   GENERATION_TYPE_INCREMENTAL = "INCREMENTAL";

    private static final String   GENERATION_TYPE_ALL         = "ALL";

    private static final String   GENERATION_TYPE_ONE         = "ONE";

    private static final String   UPDATE_TYPE_OPTION          = "u";

    private static final String   GENERATION_TYPE_OPTION      = "t";

    private static final String   ENTITY_ID_OPTION            = "e";

    private static final String   TIMESTAMP_OPTION            = "s";

    // Default values of cmdline options

    private static String         UPDATE_TYPE                 = UPDATE_TYPE_CONTENT;

    private static String         GENERATION_TYPE             = GENERATION_TYPE_INCREMENTAL;

    private static String         ENTITY_ID                   = "ALL";

    private Date 				  timeStamp			  		  = null;

    private CommandLine           cmd                         = null;

    private Map<Long, List<Item>> entityIdItemMap             = new LinkedHashMap<Long, List<Item>>();

    private Long                  lastGenerationTime          = 0l;

    private Map<Long, Entity>     entities;

    private List<Item>            items;

    private List<Source>          sources;

    private CatalogClient         csc;

    private Client                client;

    private long                  newLastGenerationTime;

    static {

        options = new Options();

        options.addOption(GENERATION_TYPE_OPTION, true, "Generation type");

        options.addOption(UPDATE_TYPE_OPTION, true, "Default is : "

                + UPDATE_TYPE);

        options.addOption(ENTITY_ID_OPTION, true, "all entities " + ENTITY_ID

                + " by default");

        options.addOption(TIMESTAMP_OPTION, true, "Manual timestamp");

    }

    public ContentGenerationUtility() throws Exception {

        csc = new CatalogClient();

        client = csc.getClient();

        sources = client.getAllSources();

    }

    /**

     * @param args

     * @throws Exception

     */

    public static void main(String[] args) throws Exception {

        ContentGenerationUtility cgu = new ContentGenerationUtility();

        // Load arguments

        cgu.loadArgs(args);

        // Call method based on arguments

        cgu.callMethod();

    }

    /**

     * Validate and set command line arguments. Exit after printing usage if

     * anything is astray

     * 

     * @param args

     *            String[] args as featured in public static void main()

     */

    private void loadArgs(String[] args) {

        CommandLineParser parser = new PosixParser();

        try {

            cmd = parser.parse(options, args);

        } catch (ParseException e) {

            log.error("Error parsing arguments", e);

            System.exit(1);

        }

        // Check for mandatory args

        if (!(cmd.hasOption(GENERATION_TYPE_OPTION) && cmd

                .hasOption(UPDATE_TYPE_OPTION))) {

            HelpFormatter formatter = new HelpFormatter();

            formatter.printHelp(COMMAND_LINE, options);

            System.exit(1);

        }

        GENERATION_TYPE = cmd.getOptionValue(GENERATION_TYPE_OPTION);

        UPDATE_TYPE = cmd.getOptionValue(UPDATE_TYPE_OPTION);

        // Look for optional args.

        if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {

            if (cmd.hasOption(ENTITY_ID_OPTION)) {

                ENTITY_ID = cmd.getOptionValue(ENTITY_ID_OPTION);

            } else {

                HelpFormatter formatter = new HelpFormatter();

                formatter.printHelp(COMMAND_LINE, options);

                System.exit(1);

            }

        }

        if (GENERATION_TYPE_INCREMENTAL.equals(GENERATION_TYPE))

        	if (cmd.hasOption(TIMESTAMP_OPTION)) {

        		SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");

        		try {

        		    timeStamp = df.parse(cmd.getOptionValue(TIMESTAMP_OPTION));

        		} catch(Exception e) {

        			HelpFormatter formatter = new HelpFormatter();

                    formatter.printHelp(COMMAND_LINE, options);

                    System.exit(1);

        		}

        	}

    }

    /**

     * Call method based on arguments

     * 

     * @throws Exception

     */

    private void callMethod() {

        boolean isSuccess = false;

        String logfile = "/tmp/content-from-cms.log";

        if (UPDATE_TYPE.equals(UPDATE_TYPE_CONTENT)) {

            logfile = "/tmp/content-from-cms.log";

            try {

                this.generateContent();

                isSuccess = true;

            } catch (Exception e) {

                log.error("Error generating content", e);

            }

        }

        if (UPDATE_TYPE.equals(UPDATE_TYPE_CATALOG)) {

            logfile = "/tmp/content-from-catalog.log";

            try {

                this.updatePrices();

                isSuccess = true;

            } catch (Exception e) {

                log.error("Error updating prices", e);

            }

        }

        GmailUtils gm = new GmailUtils();

        String[] sendTo = { "rajveer.singh@shop2020.in", "mandeep.dhir@shop2020.in", "pankaj.kankar@shop2020.in", "anupam.singh@shop2020.in", "amit.gupta@shop2020.in" };

        try {

            gm.sendSSLMessage(sendTo, "Content Generation Successful ? : "

                    + isSuccess, "Content generation completed at time : "

                    + Calendar.getInstance().getTime().toString(),

                    "build@shop2020.in", "cafe@nes", logfile);

        } catch (MessagingException e) {

            log.error("Could not send status mail", e);

        }

    }

    public boolean cleanDir(File dir, boolean deleteSelf) {

        if (dir.isDirectory()) {

            String[] children = dir.list();

            for (int i = 0; i < children.length; i++) {

                boolean success = cleanDir(new File(dir, children[i]), true);

                if (!success) {

                    return false;

                }

            }

        }

        // The directory is now empty so delete it

        if (deleteSelf) {

            return dir.delete();

        }

        return true;

    }

    private void removeOldResources() throws IOException {

        File f = new File(Utils.EXPORT_SOLR_PATH);

        if (f.exists()) {

            cleanDir(f, false);

        }

        File f1 = new File(Utils.EXPORT_ENTITIES_PATH_LOCALHOST);

        if (f1.exists()) {

            cleanDir(f1, false);

        }

        File f2 = new File(Utils.EXPORT_ENTITIES_PATH_SAHOLIC);

        if (f2.exists()) {

            cleanDir(f2, false);

        }

        File f3 = new File(Utils.EXPORT_ENTITIES_PATH_SHOP2020);

        if (f3.exists()) {

            cleanDir(f3, false);

        }

    }

    /**

     * Update the prices in the generated content

     * 

     * @throws Exception

     */

    private void updatePrices() throws Exception {

        lastGenerationTime = new Long(0);

        if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {

            items = client.getItemsByCatalogId(Long.parseLong(ENTITY_ID));

        } else {

            items = client.getAllItemsByStatus(status.ACTIVE);

            items.addAll(client.getAllItemsByStatus(status.PAUSED));

            try {

                //Generate prices and availability data for amazon

                AmazonSCDataGenerator.generatePricesAndAvailability();

            } catch (Exception e) {

            	log.info("Could not generate Amazon prices and availability", e);

            }

            // Clean up the data from the solr directories.

            removeOldResources();

        }

        // this still needs to be evolved. Must not be used.

        if (GENERATION_TYPE.equals(GENERATION_TYPE_INCREMENTAL)) {

        }

        // Populate the entityIdIemMap

        populateEntityIdItemMap();

        PriceInsertor priceInserter = new PriceInsertor();

        for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {

            long entityId = entry.getKey();

            List<Item> items = entry.getValue();

            // TODO Domain name and destination directory should be read from

            // properties file

            double minPrice = priceInserter.insertPriceInHtml(items, entityId,

                    "saholic.com", Utils.EXPORT_ENTITIES_PATH_SAHOLIC, null);

            priceInserter.insertPriceInHtml(items, entityId, "shop2020.in",

                    Utils.EXPORT_ENTITIES_PATH_SHOP2020, null);

            priceInserter.insertPriceInHtml(items, entityId, "localhost:8090",

                    Utils.EXPORT_ENTITIES_PATH_LOCALHOST, null);

            StringBuilder priceString = new StringBuilder(

                    "<field name=\"F_50002\">" + minPrice + "</field>");

            if (sources != null) {

                for (Source source : sources) {

                    minPrice = priceInserter.insertPriceInHtml(items, entityId,

                            "saholic.com", Utils.EXPORT_ENTITIES_PATH_SAHOLIC,

                            source);

                    priceInserter.insertPriceInHtml(items, entityId,

                            "shop2020.in", Utils.EXPORT_ENTITIES_PATH_SHOP2020,

                            source);

                    priceInserter.insertPriceInHtml(items, entityId,

                            "localhost:8090",

                            Utils.EXPORT_ENTITIES_PATH_LOCALHOST, source);

                    priceString.append("<field name=\"F_50002_"

                            + source.getId() + "\">" + minPrice + "</field>");

                }

            }

            priceInserter.insertPriceInSolrData(entityId,

                    priceString.toString());

        }

        priceInserter.copySolrSchemaFiles();

        synonymTitlesExporter();

        // Generate partners and json objects for phones only

        if (!GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {

            ProductListGenerator generator = new ProductListGenerator(entityIdItemMap);

            generator.generateProductsListXML();

            generator.generateProductListJavascript();

            try	{

            	generator.generateProductXMLForDisplayAds();

            } catch (Exception e) {

			}

        }

    }

    /**

     * Generates content for the specified entity embedding links to the

     * specified domain name.

     * 

     * The method will not generate content if one of the following conditions

     * is met:

     * <ol>

     * <li>The entity is not ready.

     * <li>The category has not been updated yet. (Set to -1).

     * <li>The content has not been updated.

     * </ol>

     * 

     * @throws

     */

    private void generateContent() throws Exception {

        if (GENERATION_TYPE.equals(GENERATION_TYPE_ALL)) {

            entities = CreationUtils.getEntities();

            lastGenerationTime = new Long(0);

        } else if (GENERATION_TYPE.equals(GENERATION_TYPE_ONE)) {

            entities = new HashMap<Long, Entity>();

            entities.put(Long.parseLong(ENTITY_ID),

                    CreationUtils.getEntity(Long.parseLong(ENTITY_ID)));

            lastGenerationTime = new Long(0);

        } else {

        	entities = CreationUtils.getEntities();

            //  When we read lastGenerationTime from database

            //  then only we should mark the 

            //	current time as newLastGenerationTime

            if(timeStamp == null) {

            	newLastGenerationTime = new Date().getTime();

            	lastGenerationTime = CreationUtils.getLastContentGenerationTime();

            } else {

            	lastGenerationTime = timeStamp.getTime();

            }

            log.info("lastGenerationTime: " + lastGenerationTime);

            if (lastGenerationTime == null) {

                lastGenerationTime = new Long(0);

            }

        } 

        // Filter invalid entities here

        List<Entity> validEntities = new ArrayList<Entity>();

        for (long entityID : entities.keySet()) {

            if (isValidEntity(entities.get(entityID))) {

                validEntities.add(entities.get(entityID));

            }

        }

        // Calculate comparison scores

        log.info("Calculating comparison scores");

        NewCMP cmp = new NewCMP(validEntities);

        Map<Long, Map<Long, Double>> slideScoresByEntity = cmp.getSlideScores();

        CreationUtils.storeSlideScores(slideScoresByEntity);

        // Fetch comparison statistics everyday and store them in BDB

        log.info("Fetching comparison statistics");

        ComparisonStatsFetcher csf = new ComparisonStatsFetcher();

        csf.fetchAndStoreComparisonStats();

        // Upload catalog to Google App Engine.

        if (GENERATION_TYPE.equals(GENERATION_TYPE_ALL)) {

            log.info("Uploading Catalog to Google app engine");

            List<Item> allItems = client.getAllItems(false);

            allItems.addAll(client.getAllItems(true));

            CatalogUploderToGAE catalogUploaderToGAE = new CatalogUploderToGAE();

            catalogUploaderToGAE.uploadItems(allItems);

        }

        items = client.getAllItemsByStatus(status.ACTIVE);

        items.addAll(client.getAllItemsByStatus(status.PAUSED));

        items.addAll(client.getAllItemsByStatus(status.CONTENT_COMPLETE));

        populateEntityIdItemMap();

        //FIXME Avoiding the finding of accesories, as list of categories for which we need to find accessories is hardocoded in code. 

        // We need to make that configurable. Also creating ticket to improve it.

        try{

	        log.info("Finding accessories");

	        AccessoriesFinder af = new AccessoriesFinder(entityIdItemMap.keySet());

	        Map<Long, Map<Long, List<Long>>> relatedAccessories = af.findAccessories();

	        CreationUtils.storeRelatedAccessories(relatedAccessories);

        }catch (Exception e) {

        	log.error("Error while generating accessories" + e);

		}

        log.info("Writing JSON file for special pages");

        SpecialPageJSONConvertor bjc = new SpecialPageJSONConvertor();

        bjc.writeToJSONFile(new File(Utils.EXPORT_JAVASCRIPT_CONTENT_PATH

                + "special-pages.json"));

        log.info("Generating velocity templates, images, documents etc.");

        NewVUI vui = new NewVUI(lastGenerationTime);

        for (Entity entity : validEntities) {

            log.info("Processing Entityid: " + entity.getID());

            vui.generateContentForOneEntity(entity, Utils.EXPORT_VELOCITY_PATH);

        }

        // Generate synonyms list. This will be used in PriceComparisonTool to

        // resolve the product names.

        log.info("Generating synonyms");

        SynonymExporter sx = new SynonymExporter();

        sx.storeSynonyms(validEntities);

        // Generate HTML for Site Index

        log.info("Generating HTML for Site Index");

        ProductIndexGenerator indexGenerator = new ProductIndexGenerator(validEntities);

        indexGenerator.generate();

        log.info("Generating HTML for Accessories Compatibility Index");

        CompatibleAccessoriesIndexGenerator generator = new CompatibleAccessoriesIndexGenerator();

        generator.generate();

        if (newLastGenerationTime != 0) {

            CreationUtils.storeLastContentGenerationTime(newLastGenerationTime);

        }

        log.info("Generating Solr files");

        NewIR ir = new NewIR(validEntities);

        ir.exportIRData();

        // ir.transformIrDataXMLtoSolrXML();

        ir.exportIRMetaData();

        ir.transformIrMetaDataXMLtoSolrSchemaXML();

        ir.transformIrMetaDataXMLtoSolrCatchAllXML();

        for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {

            List<Item> items = entry.getValue();

            for (Item item : items) {

                if (item.getItemStatus() == status.CONTENT_COMPLETE) {

                    item.setItemStatus(status.ACTIVE);

                    item.setStatus_description("This item is active");

                    client.updateItem(item);

                }

            }

        }

        try {

            //generate products list that is to be uploaded in Amazon.

            AmazonSCDataGenerator.generateSCProdData(validEntities, GENERATION_TYPE);

        } catch (Exception e) {

        	e.printStackTrace();

            log.info("Could not generate Amazon data", e);

        }

    }

    /**

     * Checks weather entity is valid or not. Entity will be invalid in one of

     * these cases:

     * <ol>

     * <li>The entity is not ready.

     * <li>The category has not been updated yet. (Set to -1).

     * <li>Content has not been updated after last content generation timestamp.

     * </ol>

     * 

     * @param entity

     * @return

     * @throws Exception

     */

    private boolean isValidEntity(Entity entity) throws Exception {

        ExpandedEntity expEntity = new ExpandedEntity(entity);

        EntityState state = CreationUtils.getEntityState(entity.getID());

        long categoryID = expEntity.getCategoryID();

        if (state.getStatus() != EntityStatus.READY || categoryID == -1) {

            return false;

        }

        if (state.getMerkedReadyOn().getTime() < this.lastGenerationTime) {

            return false;

        }

        return true;

    }

    private void populateEntityIdItemMap() {

        Date todate = new Date();

        Utils.info("Processing " + items.size() + " items");

        for (Item item : items) {

            Utils.info(item.getId() + ":" + item.getItemStatus() + ":" + item.getCatalogItemId());

            // TODO Can be removed as we are checking in calling function

            if (!(item.getItemStatus() == status.ACTIVE

                    || item.getItemStatus() == status.CONTENT_COMPLETE || item

                    .getItemStatus() == status.PAUSED)) {

                continue;

            }

            Utils.info(item.getStartDate() + ":" + item.getSellingPrice());

            if (todate.getTime() < item.getStartDate()

                    || item.getSellingPrice() == 0) {

                continue;

            }

            Utils.info(item.getId() + " Item is adding");

            List<Item> itemList = entityIdItemMap.get(item.getCatalogItemId());

            if (itemList == null) {

                itemList = new ArrayList<Item>();

            }

            itemList.add(item);

            entityIdItemMap.put(item.getCatalogItemId(), itemList);

        }

        Utils.info("Processing " + entityIdItemMap.size() + " entities");

        // Remove all items which have not been updated since last content

        // generation.

        List<Long> removeEntities = new ArrayList<Long>();

        for (Long entityId : entityIdItemMap.keySet()) {

            boolean isValidEntity = false;

            // If any one of the items has been updated before current

            // timestamp, than we generate content for whole entity

            for (Item item : entityIdItemMap.get(entityId)) {

                if (item.getUpdatedOn() > lastGenerationTime) {

                    isValidEntity = true;

                }

            }

            if (!isValidEntity) {

                removeEntities.add(entityId);

            }

        }

        for (Long entityId : removeEntities) {

            entityIdItemMap.remove(entityId);

        }

        Utils.info("Final valid entities to be processed: " + entityIdItemMap.size());

    }

    private void synonymTitlesExporter() {

    	SynonymExporter sx = new SynonymExporter();

        Map<Long, Map<String,List<String>>> synonyms = sx.getSynonyms();

        Map<Long, List<String>> finalsynonyms = new HashMap<Long, List<String>>();

    	for (Map.Entry<Long, List<Item>> entry : entityIdItemMap.entrySet()) {

            long entityId = entry.getKey();

	    	try{

	            String brand = "";

	            List<String> modelNameSynonyms =  new ArrayList<String>();

	            List<String> modelNumberSynonyms =  new ArrayList<String>();

	            List<String> titles = new ArrayList<String>();

	            Map<String,List<String>> synonymMap = synonyms.get(entityId);

	            if(synonymMap != null && !synonymMap.isEmpty()){

	            	if(synonymMap.get("ORIGINAL_MODEL_NAME") != null && !synonymMap.get("ORIGINAL_MODEL_NAME").isEmpty()){

	            		modelNameSynonyms.addAll(synonymMap.get("ORIGINAL_MODEL_NAME"));

	            	}

	            	if(synonymMap.get("MODEL_NAME") != null && !synonymMap.get("MODEL_NAME").isEmpty()){

	            		modelNameSynonyms.addAll(synonymMap.get("MODEL_NAME"));

	            	}

	            	if(synonymMap.get("ORIGINAL_MODEL_NUMBER") != null && !synonymMap.get("ORIGINAL_MODEL_NUMBER").isEmpty()){

	            		modelNumberSynonyms.addAll(synonymMap.get("ORIGINAL_MODEL_NUMBER"));

	            	}

	            	if(synonymMap.get("MODEL_NUMBER") != null && !synonymMap.get("MODEL_NUMBER").isEmpty()){

	            		modelNumberSynonyms.addAll(synonymMap.get("MODEL_NUMBER"));

	            	}

	            	brand = ((synonymMap.get("ORIGINAL_BRAND") != null && !synonymMap.get("ORIGINAL_BRAND").isEmpty()) ? synonymMap.get("ORIGINAL_BRAND").get(0) : "");

	            }

	            for(String model_name: modelNameSynonyms){

	            	for(String model_number: modelNumberSynonyms){

	            		String title = brand + " " + model_name + " " + model_number;

	            		title = title.replaceAll("  ", " ");

	            		titles.add(title);

	            	}

	            }

	            finalsynonyms.put(entityId, titles);

	        } catch (Exception e) {

				e.printStackTrace();

			}

    	}

    	String autosuggestFilename = Utils.EXPORT_JAVASCRIPT_CONTENT_PATH + "autosuggest.json";

        Gson gson = new Gson();

		try {

			DBUtils.store(gson.toJson(finalsynonyms), autosuggestFilename);

		} catch (Exception e) {

			e.printStackTrace();

		}

    }

}