Subversion Repositories SmartDukaan

package in.shop2020.util;

import in.shop2020.metamodel.core.Entity;

import in.shop2020.metamodel.core.EntityState;

import in.shop2020.metamodel.core.EntityStatus;

import in.shop2020.metamodel.util.CreationUtils;

import in.shop2020.metamodel.util.ExpandedEntity;

import in.shop2020.model.v1.catalog.InventoryService.Client;

import in.shop2020.model.v1.catalog.Item;

import in.shop2020.model.v1.catalog.Source;

import in.shop2020.model.v1.catalog.status;

import in.shop2020.thrift.clients.CatalogClient;

import in.shop2020.ui.util.CatalogUploderToGAE;

import in.shop2020.ui.util.ComparisonStatsFetcher;

import in.shop2020.ui.util.NewVUI;

import in.shop2020.ui.util.PriceInsertor;

import in.shop2020.ui.util.SpecialPageJSONConvertor;

import in.shop2020.utils.GmailUtils;

import java.io.File;

import java.io.IOException;

import java.util.ArrayList;

import java.util.Calendar;

import java.util.Date;

import java.util.HashMap;

import java.util.LinkedHashMap;

import java.util.List;

import java.util.Map;

import javax.mail.MessagingException;

import org.apache.commons.cli.CommandLine;

import org.apache.commons.cli.CommandLineParser;

import org.apache.commons.cli.HelpFormatter;

import org.apache.commons.cli.Options;

import org.apache.commons.cli.ParseException;

import org.apache.commons.cli.PosixParser;

public class ContentGenerationUtility {

    private static Options options = null; // Command line options

    private static final String UPDATE_TYPE_OPTION = "u";

    private static final String GENERATION_TYPE_OPTION = "t";

    private static final String ENTITY_ID_OPTION = "e";

    private static String UPDATE_TYPE = "CONTENT";

    private static String GENERATION_TYPE = "INCREMENTAL";

    private static String ENTITY_ID = "ALL";

    Map<Long, Entity> entities;

    List<Item> items;

    List<Item> contentCompleteItems  = new ArrayList<Item>();

    List<Item> phasedOutItems;

    List<Source> sources;

    CatalogClient csc;

    Client client;

    Map<Long, List<Item>> entityIdItemMap = new LinkedHashMap<Long, List<Item>>();

    private CommandLine cmd = null; // Command Line arguments

    Long lastGenerationTime;

    static{

        options = new Options();

        options.addOption(GENERATION_TYPE_OPTION, true, "Generation type");

        options.addOption(UPDATE_TYPE_OPTION, true, "Default is : " + UPDATE_TYPE);

        options.addOption(ENTITY_ID_OPTION, true, "all entities " + ENTITY_ID + " by default");

    }

    public ContentGenerationUtility() throws Exception{

        csc = new CatalogClient();

        client = csc.getClient();

        sources = client.getAllSources();

    }

    /**

     * @param args

     * @throws Exception 

     */

    public static void main(String[] args) throws Exception{

        ContentGenerationUtility cgu = new ContentGenerationUtility();

        //Load arguments

        cgu.loadArgs(args);

        //Call method based on arguments

        cgu.callMethod();

    }

	/**

     * Validate and set command line arguments.

     * Exit after printing usage if anything is astray

     * @param args String[] args as featured in public static void main()

     */

    private void loadArgs(String[] args){

        CommandLineParser parser = new PosixParser();

        try {

            cmd = parser.parse(options, args);

        } catch (ParseException e) {

            System.err.println("Error parsing arguments");

            e.printStackTrace();

            System.exit(1);

        }

        // Check for mandatory args

        if (!( cmd.hasOption(GENERATION_TYPE_OPTION)  &&  cmd.hasOption(UPDATE_TYPE_OPTION))){

            HelpFormatter formatter = new HelpFormatter();

            formatter.printHelp("java ContentGenerationUtility.class -t { ALL | INCREMENTAL | ONE } -u { CONTENT | CATALOG } -e {EntityId} ", options);

            System.exit(1);

        }

        GENERATION_TYPE = cmd.getOptionValue(GENERATION_TYPE_OPTION);

        UPDATE_TYPE = cmd.getOptionValue(UPDATE_TYPE_OPTION);

        // Look for optional args.

        if(GENERATION_TYPE.equals("ONE")){

            if (cmd.hasOption(ENTITY_ID_OPTION)){

                ENTITY_ID = cmd.getOptionValue(ENTITY_ID_OPTION);

            }else{

                HelpFormatter formatter = new HelpFormatter();

                formatter.printHelp("java ContentGenerationUtility.class -t { ALL | INCREMENTAL | ONE } -u { CONTENT | CATALOG } -e {EntityId} ", options);

                System.exit(1);

            }

        }

    }

    /**

     * Call method based on arguments

     * @throws Exception

     */

    private void callMethod(){

    	boolean isSuccess = false;

    	String logfile = "/tmp/content-from-cms.log";

    	if(UPDATE_TYPE.equals("CONTENT")){

    		logfile = "/tmp/content-from-cms.log";

    		try{

    			this.generateContent();

    			isSuccess = true;

    		}catch (Exception e) {

    			e.printStackTrace();

			}

    	}

    	if(UPDATE_TYPE.equals("CATALOG")){

    		logfile = "/tmp/content-from-catalog.log";

    		try{

    			this.updatePrices();

    			isSuccess = true;

    		}catch (Exception e) {

    			e.printStackTrace();

			}

    	}

    	GmailUtils gm = new GmailUtils();

		String[] sendTo = { "chandranshu.s@shop2020.in", "rajveer.singh@shop2020.in", "mandeep.dhir@shop2020.in" };

		try {

			gm.sendSSLMessage(sendTo, "Content Generation Successful ? : " + isSuccess, "Content generation completed at time : " + Calendar.getInstance().getTime().toString(), "build@shop2020.in", "shop2020", logfile);

		} catch (MessagingException e) {

			// TODO Auto-generated catch block

			e.printStackTrace();

		}

    }

	public boolean cleanDir(File dir, boolean deleteSelf) {

	    if (dir.isDirectory()) {

	        String[] children = dir.list();

	        for (int i=0; i<children.length; i++) {

	            boolean success = cleanDir(new File(dir, children[i]), true);

	            if (!success) {

	                return false;

	            }

	        }

	    }

	    // The directory is now empty so delete it

	    if(deleteSelf){

	    	return dir.delete();

	    }

	    return true;

	}

	private void removeOldResources() throws IOException{

		File f = new File(Utils.EXPORT_SOLR_PATH);

		if(f.exists()){

			cleanDir(f, false);

		}

		File f1 = new File(Utils.EXPORT_ENTITIES_PATH_LOCALHOST);

		if(f1.exists()){

			cleanDir(f1, false);

		}

		File f2 = new File(Utils.EXPORT_ENTITIES_PATH_SAHOLIC);

		if(f2.exists()){

			cleanDir(f2, false);

		}

		File f3 = new File(Utils.EXPORT_ENTITIES_PATH_SHOP2020);

		if(f3.exists()){

			cleanDir(f3, false);

		}

	}

    /**

     * Update the prices in the generated content

     * @throws Exception

     */

    private void updatePrices() throws Exception {

    	lastGenerationTime = new Long(0);

        if(GENERATION_TYPE.equals("ONE")) {

            items = client.getItemsByCatalogId(Long.parseLong(ENTITY_ID));

        }else{

            items = client.getAllItemsByStatus(status.ACTIVE);

            items.addAll(client.getAllItemsByStatus(status.PAUSED));

            //Clean up the data from the solr directories.

            removeOldResources();

        }

        //this still needs to be evolved. Must not be used.

        if(GENERATION_TYPE.equals("INCREMENTAL")) {

        }

        //Populate the entityIdIemMap 

        populateEntityIdItemMap();

        PriceInsertor priceInserter = new PriceInsertor();

        for(Map.Entry<Long, List<Item>> entry: entityIdItemMap.entrySet()){

        	long entityId = entry.getKey();

        	List<Item> items = entry.getValue();

        	//TODO Domain name and destination  directory should be read from properties file

        	double minPrice = priceInserter.insertPriceInHtml(items, entityId, "saholic.com", Utils.EXPORT_ENTITIES_PATH_SAHOLIC, null);

        	priceInserter.insertPriceInHtml(items, entityId, "shop2020.in", Utils.EXPORT_ENTITIES_PATH_SHOP2020, null);

        	priceInserter.insertPriceInHtml(items, entityId, "localhost:8090", Utils.EXPORT_ENTITIES_PATH_LOCALHOST, null);

        	StringBuilder priceString = new StringBuilder("<field name=\"F_50002\">"+ minPrice +"</field>"); 

        	if(sources != null){

        		for(Source source: sources){

            		minPrice = priceInserter.insertPriceInHtml(items, entityId, "saholic.com", Utils.EXPORT_ENTITIES_PATH_SAHOLIC, source);

            		priceInserter.insertPriceInHtml(items, entityId, "shop2020.in", Utils.EXPORT_ENTITIES_PATH_SHOP2020, source);

                	priceInserter.insertPriceInHtml(items, entityId, "localhost:8090", Utils.EXPORT_ENTITIES_PATH_LOCALHOST, source);

                	priceString.append("<field name=\"F_50002_" + source.getId() + "\">" + minPrice +"</field>");

            	}

        	}

        	priceInserter.insertPriceInSolrData(entityId, priceString.toString());

        }

        //Generate partners and json objects for phones only

        if(!GENERATION_TYPE.equals("ONE")) {

        	ProductListGenerator generator = new ProductListGenerator(entityIdItemMap);

			generator.generateProductsListXML();

			generator.generateProductListJavascript();

        }

    }

    /**

     * 

     * @param items

     * @return the minimum price of the items

     */

	private double getMinPrice(List<Item> items){

        double minPrice = Double.MAX_VALUE;

        for(Item item: items){

            if(minPrice > item.getSellingPrice()){

                minPrice = item.getSellingPrice();

            }

        }

        return minPrice;

    }

	/**

     * Generates content for the specified entity embedding links to the

     * specified domain name.

     * 

     * The method will not generate content if one of the following conditions is met:

     * <ol>

     * <li>The entity is not ready.

     * <li>The category has not been updated yet. (Set to -1).

     * <li>The content has not been updated.

     * </ol>

     *

     * @throws

     */

    private void generateContent() throws Exception{

        if(GENERATION_TYPE.equals("ALL")) {

        	entities = CreationUtils.getEntities();

            lastGenerationTime = new Long(0);

        }else if(GENERATION_TYPE.equals("ONE")) {

        	entities = new HashMap<Long, Entity>();

        	entities.put(Long.parseLong(ENTITY_ID), CreationUtils.getEntity(Long.parseLong(ENTITY_ID)));

            lastGenerationTime = new Long(0);   

        }else{

        	entities = CreationUtils.getEntities();

            lastGenerationTime = CreationUtils.getLastContentGenerationTime();

            if(lastGenerationTime==null){

                lastGenerationTime = new Long(0);

            }    

        }

        //Filter invalid entities here

        List<Entity> validEntities = new ArrayList<Entity>();

        for(long entityID: entities.keySet()){

        	if(isValidEntity(entities.get(entityID))){

        		validEntities.add(entities.get(entityID));

        	}

        }

        //Calculate comparison scores

        NewCMP cmp = new NewCMP(validEntities);

        Map<Long, Map<Long, Double>> slideScoresByEntity = cmp.getSlideScores();

        CreationUtils.storeSlideScores(slideScoresByEntity);

        // Fetch comparison statistics everyday and store them in BDB

        ComparisonStatsFetcher csf = new ComparisonStatsFetcher();

        csf.fetchAndStoreComparisonStats();

        // Upload catalod to Google App Engine.

        if(GENERATION_TYPE.equals("ALL")) {

            List<Item> allItems = client.getAllItems(false);

            allItems.addAll(client.getAllItems(true));

            CatalogUploderToGAE catalogUploaderToGAE = new CatalogUploderToGAE();

            catalogUploaderToGAE.uploadItems(allItems);

        }

        items = client.getAllItemsByStatus(status.ACTIVE);

        items.addAll(client.getAllItemsByStatus(status.PAUSED));

        items.addAll(client.getAllItemsByStatus(status.CONTENT_COMPLETE));

        populateEntityIdItemMap();

        AccessoriesFinder af = new AccessoriesFinder(entityIdItemMap.keySet());

		Map<Long, Map<Long, List<Long>>> relatedAccessories = af.findAccessories();

		CreationUtils.storeRelatedAccessories(relatedAccessories);

        SpecialPageJSONConvertor bjc = new SpecialPageJSONConvertor();

        bjc.writeToJSONFile(new File(Utils.EXPORT_JAVASCRIPT_CONTENT_PATH + "special-pages.json"));

        NewVUI vui = new NewVUI(lastGenerationTime);

        for(Entity entity: validEntities){

        		vui.generateContentForOneEntity(entity, Utils.EXPORT_VELOCITY_PATH);

        }

        CreationUtils.storeLastContentGenerationTime((new Date()).getTime());

        NewIR ir = new NewIR(validEntities);

        ir.exportIRData();

        //ir.transformIrDataXMLtoSolrXML();

        ir.exportIRMetaData();

        ir.transformIrMetaDataXMLSolrSchemaXML();

        // Generate synonyms list. This will be used in PriceComparisonTool to resolve the product names.

        SynonymExporter sx = new SynonymExporter(validEntities);

        sx.storeSynonyms();

        for(Map.Entry<Long, List<Item>> entry: entityIdItemMap.entrySet()){

        	List<Item> items = entry.getValue();

        	for(Item item: items){

        		if(item.getItemStatus()==status.CONTENT_COMPLETE){

                    item.setItemStatus(status.ACTIVE);

                    item.setStatus_description("This item is active");

                    client.updateItem(item);

        		}

        	}

        }

    }

    /**

     * Checks weather entity is valid or not. Entity will be invalid in one of these cases:

     * <ol>

     * <li>The entity is not ready.

     * <li>The category has not been updated yet. (Set to -1).

     * <li>Content has not been updated after last content generation timestamp.

     * </ol>

     * 

     * @param entity

     * @return

     * @throws Exception

     */

    private boolean isValidEntity(Entity entity) throws Exception{

        ExpandedEntity expEntity = new ExpandedEntity(entity);

        EntityState state = CreationUtils.getEntityState(entity.getID());

        long categoryID = expEntity.getCategoryID();

        if(state.getStatus() != EntityStatus.READY ||  categoryID == -1){

            return false;

        }

        if(state.getMerkedReadyOn().getTime() < this.lastGenerationTime){

            return false;

        }

        return true;

    }

    private void populateEntityIdItemMap(){

        Date todate = new Date();

        for(Item item: items){

            //TODO Can be removed as we are checking in calling function

            if(!(item.getItemStatus()==status.ACTIVE || item.getItemStatus()==status.CONTENT_COMPLETE || item.getItemStatus() == status.PAUSED)){

                continue;

            }

            if(todate.getTime() < item.getStartDate() ||  item.getSellingPrice() == 0){

                continue;

            }

            List<Item> itemList = entityIdItemMap.get(item.getCatalogItemId());

            if(itemList == null){

                itemList = new ArrayList<Item>();

            }

            itemList.add(item);

            entityIdItemMap.put(item.getCatalogItemId(), itemList);

        }

        //Remove all items which have not been updated since last content generation.

        List<Long> removeEntities = new ArrayList<Long>();

        for(Long entityId:entityIdItemMap.keySet()){

            boolean isValidEntity = false;

            //If any one of the items has been updated before current timestamp, than we generate content for whole entity

            for(Item item: entityIdItemMap.get(entityId)){

                if(item.getUpdatedOn() > lastGenerationTime){

                    isValidEntity = true;

                }

            }

            if(!isValidEntity){

                removeEntities.add(entityId);

            }

        }

        for(Long entityId: removeEntities){

            entityIdItemMap.remove(entityId);

        }

    }

}