Subversion Repositories SmartDukaan

/**

 * 

 */

package in.shop2020.serving.services;

import in.shop2020.config.ConfigException;

import in.shop2020.serving.controllers.SearchController;

import in.shop2020.serving.utils.Utils;

import in.shop2020.thrift.clients.config.ConfigClient;

import java.io.UnsupportedEncodingException;

import java.net.URLEncoder;

import java.util.ArrayList;

import java.util.Arrays;

import java.util.Collections;

import java.util.HashMap;

import java.util.HashSet;

import java.util.Iterator;

import java.util.LinkedHashMap;

import java.util.LinkedList;

import java.util.List;

import java.util.Map;

import java.util.Set;

import java.util.TreeMap;

import java.util.regex.Matcher;

import java.util.regex.Pattern;

import javax.xml.xpath.XPath;

import javax.xml.xpath.XPathConstants;

import javax.xml.xpath.XPathExpressionException;

import javax.xml.xpath.XPathFactory;

import org.apache.commons.collections.ListUtils;

import org.apache.log4j.Logger;

import org.w3c.dom.Node;

import org.w3c.dom.NodeList;

import org.xml.sax.InputSource;

/**

 * @author rajveer

 *

 */

public class SolrSearchService {

	/**

	 * 

	 */

	private static Logger log = Logger.getLogger(Class.class);

	private HashMap<String, Double> dynamicPriceMap = null;

	/**

	 * 

	 */

	public static final String SOLR_URL;

	private static final Pattern FACET_PATTERN = Pattern.compile("(?=(F_\\d{5}))");

	private static final Map<String, List<String>> SORTED_FACET_VALUE_MAP = Collections.unmodifiableMap(

			new HashMap<String, List<String>>(){

				/**

				 * 

				 */

				private static final long serialVersionUID = 1L;

				{

					put("F_50007", Arrays.asList("Upto 2 Mpx", "2 - 5 Mpx", "5 - 10 Mpx", "Above 10 Mpx"));

					put("F_50024", Arrays.asList("Upto 10 Mpx", "10 - 12 Mpx", "12 - 15 Mpx", "15 - 18 Mpx", "Above 18 Mpx"));

					put("F_50025", Arrays.asList("Upto 4x", "4 - 6x", "6 - 10x", "10 - 14x", "14 - 18x", "Above 18x"));

					put("F_50026", Arrays.asList("Below 2 in.", "2 to 2.9 in.", "3 to 3.9 in.", "Above 4 in."));

					put("F_50032", Arrays.asList("Below 3 in.", "3 to 3.9 in.", "4 to 4.9 in.", "Above 5 in."));

					put("F_50027", Arrays.asList("Upto 10 Mpx", "10 - 15 Mpx", "15 - 20 Mpx", "Above 20 Mpx"));

				}

		});

	public static final Map<String, List<String>> CATEGORY_FACET_MAP = Collections.unmodifiableMap(

			new HashMap<String, List<String>>(){

				/**

				 * 

				 */

				private static final long serialVersionUID = 1L;

				//Data Connectivity, Camera Resolution,Operating System,Screen Size

				List<String> mobileFacets = Arrays.asList("F_50038", "F_50006", "F_50007", "F_50031", "F_50032");

				//Camera Resolution,Operating System,Screen Size

				List<String> tabletFacets = Arrays.asList("F_50037","F_50036", "F_50035", "F_50034");

				//Operating System, Processor, Storage, RAM, Screen Size

				List<String> laptopFacets = Arrays.asList("F_50013", "F_50014", "F_50015","F_50017", "F_50033");

				//Resolution, Optical Zoon, Display Size

				List<String> compactCameras = Arrays.asList("F_50024", "F_50025", "F_50026");

				//Display Size

				List<String> dslrCameras = Arrays.asList("F_50026");

				//Capacity, Class

				List<String> memoryCards = Arrays.asList("F_50018", "F_50019");

				//Capacity

				List<String> penDrives = Arrays.asList("F_50020");

				//Capacity, Type, Interface

				List<String> externalHardDisks = Arrays.asList("F_50021", "F_50022", "F_50023");

				{					

					put(SearchController.getCategoryLabel(10001l), mobileFacets);

					put(SearchController.getCategoryLabel(10006l), mobileFacets);

					put(SearchController.getCategoryLabel(10009l), tabletFacets);

					put(SearchController.getCategoryLabel(10010l), tabletFacets);

					put(SearchController.getCategoryLabel(10013l), memoryCards);

					put(SearchController.getCategoryLabel(10017l), penDrives);

					put(SearchController.getCategoryLabel(10049l), laptopFacets);

					put(SearchController.getCategoryLabel(10050l), laptopFacets);

					put(SearchController.getCategoryLabel(10073l), externalHardDisks);

					put(SearchController.getCategoryLabel(11002l), compactCameras);

					put(SearchController.getCategoryLabel(11003l), dslrCameras);

				}

			});

	static {

		String solr_url = null;

		try {

			solr_url = ConfigClient.getClient().get("solr_url");

		}catch(ConfigException cex){

		    log.error("Unable to get the solr URL from the config server. Setting the default value.", cex);

			solr_url = "http://localhost:8983/solr/select/";

		}

		SOLR_URL = solr_url;

	}

	/**

	 * 

	 */

	private XPath xpath;

	/**

	 * 

	 */

	private InputSource inputSource;

	Map<String,Map<String,Integer>> facetMap;

	private String 	query;

	List<String> resultMap;

	long numberOfResults=0;

	String priceFacetName = "F_50002";

	List<String> filtrableFacets = new ArrayList<String>(Utils.rootfacetDefIDs);

	/**

	 * 

	 * @param query

	 * @param facetDefinitionIDs

	 */

	public SolrSearchService(String query, String[] facetqueries, long start, long rows,  Double minPrice, Double maxPrice, String sortOrder, long sourceId) {

		this.query = query;

		List<String> facetsQueried = new ArrayList<String>();

		if(sourceId != -1){

			priceFacetName = priceFacetName + "_" + sourceId;

		}

		setFilterableFacets(facetqueries);

		this.filtrableFacets.remove(Utils.availabilityFacet);

		this.filtrableFacets.add(Utils.availabilityFacet);

		this.xpath = XPathFactory.newInstance().newXPath();

		query = query.trim().replaceAll("\\s+", " ");

    	log.info("query=" + query);

		String uri = SOLR_URL + "?wt=xml&q=" + this.query;

		uri += "&stats=on&stats.field=" + priceFacetName;

		if(sortOrder != null){

			//replace the price facet name, so that it can pick price for the source.

			sortOrder = sortOrder.replace("F_50002", priceFacetName);

			uri += "&sort=" + sortOrder;

		}

		if(facetqueries != null) {

			//sorting will guarantee all similar facets together so that we can assume or between all similar items without fail.

			Arrays.sort(facetqueries);

			String fq="";

			for(int i=0; i<facetqueries.length; i++) {

				String value = "";

				String[] tokens = facetqueries[i].split(":");

				try {

					value = URLEncoder.encode(tokens[1], "UTF-8");

				} catch (UnsupportedEncodingException e) {

					// TODO Auto-generated catch block

					e.printStackTrace();

				}

				if(facetsQueried.contains(tokens[0])) {

					uri += " OR  " + tokens[0] + ":\"" + value + "\"";

				} else {

					fq = "{!tag=dt" + facetsQueried.size() + "}";

					facetsQueried.add(tokens[0]);

					fq +=  tokens[0] + ":\"" + value + "\"";

					uri += "&fq=" + fq;

				}

			}

		}

		String minString = "0";

		String maxString = "*";  

		if(minPrice != null || maxPrice != null){

			try {

				dynamicPriceMap = getPriceStatsMap(new InputSource(uri)); 

			} catch (Exception e){

				e.printStackTrace();

			}

			if(minPrice != null){

				minString = minPrice.toString();

			}

			if(maxPrice != null){

				maxString = maxPrice.toString();

			}

		}

		uri += "&fq=" + priceFacetName + ":["+  minString + "%20" + maxString + "]";

		uri += "&fl=ID,Name&facet=true&start=" + start + "&rows=" + rows + "&facet.mincount=1";

		for(String facetDefinitionID : filtrableFacets) {

				if(facetsQueried.contains(facetDefinitionID)){

					uri += "&facet.field={!ex=dt" + facetsQueried.indexOf(facetDefinitionID)+ "}"+ facetDefinitionID; 

				} else {

					uri += "&facet.field=" + facetDefinitionID;

				}

		}

		log.info("uri=" + uri);

		this.inputSource = new InputSource(uri);

		this.facetMap = getFacetMap();

	}

	@SuppressWarnings("unchecked")

	private void setFilterableFacets(String[] facetqueries) {

		List<String> queriedFacets = getAllMatches(this.query);

		if(facetqueries != null && facetqueries.length!=0) {

			String facetString = Arrays.toString(facetqueries);

			List<String> filteredFacets = getAllMatches(facetString);

			if(filteredFacets.contains("F_50011")){

				for(String facetQuery : facetqueries) {

					if(facetQuery.contains("F_50011")){

						String facetVal = facetQuery.split(":")[1];

						if(CATEGORY_FACET_MAP.containsKey(facetVal)){

							this.filtrableFacets.addAll(CATEGORY_FACET_MAP.get(facetVal));

							return;

						} else {

							break;

						}

					}

				}

			}

			if(filteredFacets.contains("F_50010")){

				for(String facetQuery : facetqueries) {

					if(facetQuery.contains("F_50010")){

						String facetVal = facetQuery.split(":")[1];

						if(CATEGORY_FACET_MAP.containsKey(facetVal)){

							return;

						} else {

							break;

						}

					}

				}

			}

		}

		if(queriedFacets.contains("F_50011")) {

			String facetVal = this.query.split("F_50011:")[1].split("&")[0].replaceAll("[\"()]", "");

			if (facetVal.contains(" OR ")) {

				return;

			} else if(CATEGORY_FACET_MAP.containsKey(facetVal)){

				//facetVal = facetVal.split("&")[0].replaceAll("[\"()]", "");

					this.filtrableFacets.addAll(CATEGORY_FACET_MAP.get(facetVal));

					return;

			} else {

				return;

			}

		}

		if(queriedFacets.contains("F_50010")){

			String facetVal = this.query.split("F_50010:")[1].split("&")[0].replaceAll("[\"()]", "");

			if (facetVal.contains(" OR ")) {

				return;

			} else if(CATEGORY_FACET_MAP.containsKey(facetVal)){

					this.filtrableFacets.addAll(CATEGORY_FACET_MAP.get(facetVal));

					return;

			} else {

				return;

			}

		}

	}

	public List<String> getFilterableFacets() {

		return this.filtrableFacets; 

	}

	public Map<String,Map<String,Integer>> removeUnwantedFacets(Map<String,Map<String,Integer>> facetMap, long numberOfResults){

		Set<String> facetsInQuery = new HashSet<String>(getAllMatches(this.query));

		Map<String,Map<String,Integer>> tempFacets = new TreeMap<String, Map<String,Integer>>(); 

		for(String facet : facetMap.keySet()){

			if(facetMap.get(facet).size() > 0 && !facetsInQuery.contains(facet)){

				Map<String,Integer> tempMap = new LinkedHashMap<String, Integer>();

				for(String facetValueName : facetMap.get(facet).keySet()){

					//if(facetMap.get(facet).get(facetValueName) != 0 && facetMap.get(facet).get(facetValueName) != numberOfResults){

						tempMap.put(facetValueName, facetMap.get(facet).get(facetValueName));

					//}

				}

				if(!tempMap.isEmpty()){

					tempFacets.put(facet, tempMap);

				}

			}	

		}

		/*if(tempFacets.containsKey("F_50010")){

			tempFacets.remove("F_50011");

		}*/

		return tempFacets;

	}

	public Map<String,Integer> getFacetDetails(String facetName){

		if(facetMap != null){

			return facetMap.get(facetName);

		}else{

			return null;

		}

	}

	public Map<String,Map<String,Integer>> getFacetMap() {

		facetMap = new TreeMap<String,Map<String,Integer>>();

		String facetNamePath = "/response/lst/lst[@name = 'facet_fields']/lst";

		NodeList nodes = null;

		try {

			nodes = (NodeList) this.xpath.evaluate(facetNamePath, this.inputSource, XPathConstants.NODESET);

		}

		catch (XPathExpressionException xpee) {

			return null;

		}

		if(nodes.getLength() == 0) {

			return null;

		}

		NodeList subNodes = null;

		for(int i=0; i<nodes.getLength(); i++) {

			Node node = nodes.item(i);

			String facetName = node.getAttributes().getNamedItem("name").getNodeValue();

			subNodes = node.getChildNodes();

			Map<String,Integer> facetValueCountMap = new LinkedHashMap<String,Integer>();

			for(int j=0; j<subNodes.getLength(); j++) {

				Node subNode = subNodes.item(j);

				facetValueCountMap.put(subNode.getAttributes().getNamedItem("name").getNodeValue(), Integer.parseInt(subNode.getTextContent()));

			}

			if(SORTED_FACET_VALUE_MAP.containsKey(facetName)){

				List<String> orderedValues = SORTED_FACET_VALUE_MAP.get(facetName);

				Map<String, Integer> sortedMap = new LinkedHashMap<String, Integer>();

			    for (Iterator<String> it = orderedValues.iterator(); it.hasNext();) {

			    	String val = it.next();

			        if(facetValueCountMap.containsKey(val)) {

			        	sortedMap.put(val, facetValueCountMap.get(val));

			        }

			    }

			    facetMap.put(facetName, sortedMap);

			} else {

				facetMap.put(facetName, facetValueCountMap);

			}

		}

		this.numberOfResults  = this.getTotalResults();

		facetMap = removeUnwantedFacets(facetMap, numberOfResults);

		return facetMap;

	}

	public List<String> getResultMap() {

		resultMap = new LinkedList<String>();

		String resultDocsPath = "/response/result/doc";

		NodeList nodes = null;

		try {

			nodes = (NodeList) this.xpath.evaluate(resultDocsPath, this.inputSource, XPathConstants.NODESET);

		}

		catch (XPathExpressionException xpee) {

			return null;

		}

		if(nodes.getLength() == 0) {

			return null;

		}

		for(int i=0; i<nodes.getLength(); i++) {

			Node node = nodes.item(i);

			String docID = node.getFirstChild().getTextContent();

			resultMap.add(docID);	

 		}

		return resultMap;

	}

	public HashMap<String, Double> getPriceStatsMap() {

		return this.getPriceStatsMap(this.inputSource);

	}

	public HashMap<String, Double> getPriceStatsMap(InputSource inputSource) {

		HashMap<String, Double> priceStatsMap = new HashMap<String, Double>();

		String resultDocsPath = "/response/lst[@name = 'stats']/lst[@name = 'stats_fields']/lst[@name = '" + priceFacetName + "']";

		NodeList nodes = null;

		try {

			nodes = (NodeList) this.xpath.evaluate(resultDocsPath, inputSource, XPathConstants.NODESET);

		}

		catch (XPathExpressionException xpee) {

			return null;

		}

		if(nodes.getLength() == 0) {

			return null;

		}

		NodeList subNodes = nodes.item(0).getChildNodes();

		for(int i=0; i<subNodes.getLength(); i++) {

			Node node = subNodes.item(i);

			String parameter = node.getAttributes().getNamedItem("name").getNodeValue();

			String value = node.getTextContent();

			priceStatsMap.put(parameter, Double.parseDouble(value));	

 		}

		return priceStatsMap;

	}

	public HashMap<String,Integer> getRangeQueryResultMap() {

		HashMap<String, Integer> rangeQueryResultMap = new HashMap<String,Integer>();

		String resultDocsPath = "/response/lst[@name = 'facet_counts']/lst[@name = 'facet_queries']/int";

		NodeList nodes = null;

		try {

			nodes = (NodeList) this.xpath.evaluate(resultDocsPath, this.inputSource, XPathConstants.NODESET);

		}

		catch (XPathExpressionException xpee) {

			return null;

		}

		if(nodes.getLength() == 0) {

			return null;

		}

		for(int i=0; i<nodes.getLength(); i++) {

			Node node = nodes.item(i);

			String query = node.getAttributes().getNamedItem("name").getNodeValue();

			String docCount = node.getTextContent();

			rangeQueryResultMap.put(query,Integer.parseInt(docCount));	

 		}

		return rangeQueryResultMap;

	}

	/**

	 * 

	 */

	public long getTotalResults(){

		String resultDocsPath = "/response/result";

		NodeList nodes = null;

		try {

			nodes = (NodeList) this.xpath.evaluate(resultDocsPath, this.inputSource, XPathConstants.NODESET);

		}

		catch (XPathExpressionException xpee) {

			return 0;

		}

		Node node = nodes.item(0);

		return Long.parseLong(node.getAttributes().getNamedItem("numFound").getNodeValue());

	}

		/**

	 * 

	 * @return

	 */

	public long[] getResultEntityIDs() {

		String expression = "/response/result/doc/long";

		NodeList nodes = null;

		try {

			nodes = (NodeList) this.xpath.evaluate(expression, this.inputSource,

					XPathConstants.NODESET);

		} 

		catch(XPathExpressionException xpee) {

			return null;

		}

		if(nodes.getLength() == 0) {

			return null;

		}

		long[] values = new long[nodes.getLength()];

		for(int i=0; i<nodes.getLength(); i++) {

			Node node = nodes.item(i);

			String value = node.getTextContent();

			values[i] = Long.parseLong(value);

 		}

		return values;

	}

	/**

	 * 

	 * @return

	 */

	public String[] getResultCategoryNames() {

		String expression = "/response/lst/lst[@name = 'facet_fields']/";

		expression += "lst[@name = 'Category']/int/@name";

		NodeList nodes = null;

		try {

			nodes = (NodeList) this.xpath.evaluate(expression, 

				this.inputSource, XPathConstants.NODESET);

		}

		catch (XPathExpressionException xpee) {

			return null;

		}

		if(nodes.getLength() == 0) {

			return null;

		}

		String[] values = new String[nodes.getLength()];

		for(int i=0; i<nodes.getLength(); i++) {

			Node node = nodes.item(i);

			values[i] = node.getTextContent();

 		}

		return values;

	}

	/**

	 * 

	 * @return

	 */

	public int[] getResultCategoryCounts() {

		String expression = "/response/lst/lst[@name = 'facet_fields']/";

		expression += "lst[@name = 'Category']/int";

		NodeList nodes = null;

		try {

			nodes = (NodeList) this.xpath.evaluate(expression, 

				this.inputSource, XPathConstants.NODESET);

		}

		catch (XPathExpressionException xpee) {

			return null;

		}

		if(nodes.getLength() == 0) {

			return null;

		}

		int[] values = new int[nodes.getLength()];

		for(int i=0; i<nodes.getLength(); i++) {

			Node node = nodes.item(i);

			values[i] = Integer.parseInt(node.getTextContent());

 		}

		return values;

	}

	/**

	 * 

	 * @return

	 */

	public String[]  getResultEntityNames() {

		String expression = "/response/result/doc/str";

		NodeList nodes = null;

		try {

			nodes = (NodeList) this.xpath.evaluate(expression, this.inputSource,

					XPathConstants.NODESET);

		} 

		catch(XPathExpressionException xpee) {

			return null;

		}

		if(nodes.getLength() == 0) {

			return null;

		}

		String[] values = new String[nodes.getLength()];

		for(int i=0; i<nodes.getLength(); i++) {

			Node node = nodes.item(i);

			String value = node.getTextContent();

			values[i] = value;

 		}

		return values;

	}

	/**

	 * 

	 * @param facetDefinitionID

	 * @return

	 */

	public String[] getFacetValues(String facetDefinitionID) {

		String expression = "/response/lst/lst[@name = 'facet_fields']/";

		expression += "lst[@name = '"+ facetDefinitionID +"']/int/@name";

		NodeList nodes = null;

		try {

			nodes = (NodeList) this.xpath.evaluate(expression, 

				this.inputSource, XPathConstants.NODESET);

		}

		catch (XPathExpressionException xpee) {

			return null;

		}

		if(nodes.getLength() == 0) {

			return null;

		}

		String[] values = new String[nodes.getLength()];

		for(int i=0; i<nodes.getLength(); i++) {

			Node node = nodes.item(i);

			values[i] = node.getTextContent();

		}

		return values;

	}

	/**

	 * 

	 * @param facetDefinitionID

	 * @return

	 */

	public String[] getFacetCounts(String facetDefinitionID) {

		String expression = "/response/lst/lst[@name = 'facet_fields']/";

		expression += "lst[@name = '" + facetDefinitionID + "']/int";

		NodeList nodes = null;

		try {

			nodes = (NodeList) this.xpath.evaluate(expression, 

				this.inputSource, XPathConstants.NODESET);

		}

		catch (XPathExpressionException xpee) {

			return null;

		}

		if(nodes.getLength() == 0) {

			return null;

		}

		String[] values = new String[nodes.getLength()];

		for(int i=0; i<nodes.getLength(); i++) {

			Node node = nodes.item(i);

			values[i] = node.getTextContent();

 		}

		return values;

	}

	public static void main(String[] args){

		/*

    	// Hard coded for now

    	String[] facetDefIDs = new String[] {"F_50001", "F_50002", "F_50003", "F_50004", "F_50005", "F_50006", "F_50007", "F_50008", "F_50009"};

    	// Hard-coded for now

    	String[] facetLabels = new String[] {

	    	"Brand", "Price","Form Factor", "Carry In Pocket", "Cellular Technologies", 

	    	"Data Connectivity", "Camera Resolution", "Built-in Memory", 

	    	"Talk time"

    	};

		 */

    	String[] facetDefIDs = new String[] {"Category","F_50002","F_50001",  "F_50006", "F_50007" };

    	//String[] facetLabels = new String[] {"Category","Price", "Brand", "Data Connectivity", "Camera Resolution"	};

    	String[] fqrys = {};

		SolrSearchService search = new SolrSearchService("nokia", fqrys, 0 , 20, null, null, null, -1);

    	long[] entityIDs = search.getResultEntityIDs();

    	log.info("entityIDs=" + Arrays.toString(entityIDs));

    	String[] entityNames = search.getResultEntityNames();

    	log.info("entityNames=" + Arrays.toString(entityNames));

    	search.getFacetMap();

    	search.getResultMap();

    	search.getRangeQueryResultMap();

    	search.getPriceStatsMap(new InputSource());

    	search.getTotalResults();

       	for (int i=0; i<facetDefIDs.length; i++) {

       		search.getFacetCounts(facetDefIDs[i]);

       		search.getFacetValues(facetDefIDs[i]);

       	}

	}

    public static List<String> getAllMatches(String text) {

        List<String> matches = new ArrayList<String>();

        Matcher m = FACET_PATTERN.matcher(text);

        while(m.find()) {

            matches.add(m.group(1));

        }

        return matches;

    }

    public Map<String, Double> getDynamicPriceMap() {

    	return this.dynamicPriceMap;

    }

}