Subversion Repositories SmartDukaan

/**
 * 
 */
package in.shop2020.serving.service;


import in.shop2020.config.ConfigException;
import in.shop2020.serving.controllers.SearchController;
import in.shop2020.serving.utils.Utils;
import in.shop2020.thrift.clients.config.ConfigClient;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;

import org.apache.commons.collections.ListUtils;
import org.apache.log4j.Logger;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;


/**
 * @author rajveer
 *
 */
public class SearchService {
    /**
     * 
     */
    private static Logger log = Logger.getLogger(Class.class);
    private HashMap<String, Double> dynamicPriceMap = null;
    
    /**
     * 
     */
    public static final String SOLR_URL;
    
    private static final Pattern FACET_PATTERN = Pattern.compile("(?=(F_\\d{5}))");
    
    private static final Map<String, List<String>> SORTED_FACET_VALUE_MAP = Collections.unmodifiableMap(
            new HashMap<String, List<String>>(){
                /**
                 * 
                 */
                private static final long serialVersionUID = 1L;
                {
                    put("F_50007", Arrays.asList("Upto 2 Mpx", "2 - 5 Mpx", "5 - 10 Mpx", "10 Mpx and above"));
                    put("F_50024", Arrays.asList("Upto 10 Mpx", "10 - 12 Mpx", "12 - 15 Mpx", "15 - 18 Mpx", "18 Mpx and above"));
                    put("F_50025", Arrays.asList("Upto 4x", "4 - 6x", "6 - 10x", "10 - 14x", "14 - 18x", "18x and above"));
                    put("F_50026", Arrays.asList("Below 2 in.", "2 to 2.9 in.", "3 to 3.9 in.", "4 in. and above"));
                    put("F_50032", Arrays.asList("Below 3 in.", "3 to 3.9 in.", "4 to 4.9 in.", "5 in. and above"));
                    put("F_50027", Arrays.asList("Upto 10 Mpx", "10 - 15 Mpx", "15 - 20 Mpx", "20 Mpx and above"));
                }
        });
    public static final Map<String, List<String>> CATEGORY_FACET_MAP = Collections.unmodifiableMap(
            new HashMap<String, List<String>>(){
                /**
                 * 
                 */
                private static final long serialVersionUID = 1L;
                //Data Connectivity, Camera Resolution,Operating System,Screen Size
                List<String> mobileFacets = Arrays.asList("F_50006", "F_50007", "F_50031", "F_50032");
                //Camera Resolution,Operating System,Screen Size
                List<String> tabletFacets = Arrays.asList("F_50036", "F_50035", "F_50034");
                //Operating System, Processor, Storage, RAM, Screen Size
                List<String> laptopFacets = Arrays.asList("F_50013", "F_50014", "F_50015","F_50017", "F_50033");
                //Resolution, Optical Zoon, Display Size
                List<String> compactCameras = Arrays.asList("F_50024", "F_50025", "F_50026");
                //Display Size
                List<String> dslrCameras = Arrays.asList("F_50026");
                //Capacity, Class
                List<String> memoryCards = Arrays.asList("F_50018", "F_50019");
                //Capacity
                List<String> penDrives = Arrays.asList("F_50020");
                //Capacity, Type, Interface
                List<String> externalHardDisks = Arrays.asList("F_50021", "F_50022", "F_50023");
                {                   
                    put(SearchController.getCategoryLabel(10001l), mobileFacets);
                    put(SearchController.getCategoryLabel(10006l), mobileFacets);
                    put(SearchController.getCategoryLabel(10002l), mobileFacets);
                    put(SearchController.getCategoryLabel(10003l), mobileFacets);
                    put(SearchController.getCategoryLabel(10004l), mobileFacets);
                    put(SearchController.getCategoryLabel(10005l), mobileFacets);
                    put(SearchController.getCategoryLabel(10006l), mobileFacets);
                    put(SearchController.getCategoryLabel(10009l), tabletFacets);
                    put(SearchController.getCategoryLabel(10010l), tabletFacets);
                    put(SearchController.getCategoryLabel(10013l), memoryCards);
                    put(SearchController.getCategoryLabel(10017l), penDrives);
                    put(SearchController.getCategoryLabel(10049l), laptopFacets);
                    put(SearchController.getCategoryLabel(10050l), laptopFacets);
                    put(SearchController.getCategoryLabel(10073l), externalHardDisks);
                    put(SearchController.getCategoryLabel(11002l), compactCameras);
                    put(SearchController.getCategoryLabel(11003l), dslrCameras);
                }
            });
    static {
        String solr_url = null;
        try {
            solr_url = ConfigClient.getClient().get("solr_url");
        }catch(ConfigException cex){
            log.error("Unable to get the solr URL from the config server. Setting the default value.", cex);
            solr_url = "http://localhost:8983/solr/select/";
        }
        SOLR_URL = solr_url;
    }
    
    /**
     * 
     */
    private XPath xpath;
    
    /**
     * 
     */
    private InputSource inputSource;
    
    Map<String,Map<String,Integer>> facetMap;
    
    private String  query;
    
    List<String> resultMap;
    
    long numberOfResults=0;
    
    String priceFacetName = "F_50002";
    
    List<String> filtrableFacets;
    
    /**
     * 
     * @param query
     * @param facetDefinitionIDs
     */
    public SearchService(String query, String[] facetqueries, long start, long rows,  Double minPrice, Double maxPrice, String sortOrder, long sourceId) {
        this.query = query;
        
        List<String> facetsQueried = new ArrayList<String>();
        if(sourceId != -1){
            priceFacetName = priceFacetName + "_" + sourceId;
        }
        
        setFilterableFacets(facetqueries);
        
        
        this.xpath = XPathFactory.newInstance().newXPath();
        
        query = query.trim().replaceAll("\\s+", " ");
        log.info("query=" + query);
        
        String uri = SOLR_URL + "?wt=xml&q=" + query;
        
        uri += "&stats=on&stats.field=" + priceFacetName;
        
        if(sortOrder != null){
            //replace the price facet name, so that it can pick price for the source.
            sortOrder = sortOrder.replace("F_50002", priceFacetName);
            uri += "&sort=" + sortOrder;
        }
        
        if(facetqueries != null) {
            //sorting will guarantee all similar facets together so that we can assume or between all similar items without fail.
            Arrays.sort(facetqueries);
            String fq="";
            for(int i=0; i<facetqueries.length; i++) {
                String[] tokens = facetqueries[i].split(":");
                if(facetsQueried.contains(tokens[0])) {
                    uri += " OR ";
                    if(facetqueries[i].contains(" ")){
                        uri +=  "\"" + tokens[1] + "\"";
                    }else{
                        uri += facetqueries[i];
                    }
                    
                } else {
                    fq = "{!tag=dt" + facetsQueried.size() + "}";
                    facetsQueried.add(tokens[0]);
                    if(facetqueries[i].contains(" ") && !(facetqueries[i].contains(" OR "))){
                        fq +=  tokens[0] + ":\"" + tokens[1] + "\"";
                    }else{
                        fq += facetqueries[i] + "";
                    }
                    uri += "&fq=" + fq;
                }
            }
        }
        String minString = "0";
        String maxString = "*";  
        if(minPrice != null || maxPrice != null){
            try {
                dynamicPriceMap = getPriceStatsMap(new InputSource(uri)); 
            } catch (Exception e){
                e.printStackTrace();
            }
            if(minPrice != null){
                minString = minPrice.toString();
            }
            if(maxPrice != null){
                maxString = maxPrice.toString();
            }
        }
        uri += "&fq=" + priceFacetName + ":["+  minString + " " + maxString + "]";
        uri += "&fl=ID,Name&facet=true&start=" + start + "&rows=" + rows + "&facet.mincount=1";
        for(String facetDefinitionID : filtrableFacets) {
                if(facetsQueried.contains(facetDefinitionID)){
                    uri += "&facet.field={!ex=dt" + facetsQueried.indexOf(facetDefinitionID)+ "}"+ facetDefinitionID; 
                } else {
                    uri += "&facet.field=" + facetDefinitionID;
                }
        }
        log.info("uri=" + uri);
        
        this.inputSource = new InputSource(uri);

        this.facetMap = getFacetMap();
    }

    @SuppressWarnings("unchecked")
    private void setFilterableFacets(String[] facetqueries) {
        List<String> queriedFacets = getAllMatches(this.query);
        if(facetqueries != null) {
            String facetString = Arrays.toString(facetqueries);
            List<String> filteredFacets = getAllMatches(facetString);
            if(filteredFacets.contains("F_50011")){
                for(String facetQuery : facetqueries) {
                    if(facetQuery.contains("F_50011")){
                        String facetVal = facetQuery.split(":")[1];
                        if(CATEGORY_FACET_MAP.containsKey(facetVal)){
                            this.filtrableFacets = ListUtils.sum(Utils.rootfacetDefIDs, CATEGORY_FACET_MAP.get(facetVal));
                            return;
                        } else {
                            break;
                        }
                    }
                }
            }
            if(filteredFacets.contains("F_50010")){
                for(String facetQuery : facetqueries) {
                    if(facetQuery.contains("F_50010")){
                        String facetVal = facetQuery.split(":")[1];
                        if(CATEGORY_FACET_MAP.containsKey(facetVal)){
                            this.filtrableFacets = ListUtils.sum(Utils.rootfacetDefIDs, CATEGORY_FACET_MAP.get(facetVal));
                            return;
                        } else {
                            break;
                        }
                    }
                }
            }
        }
        if(queriedFacets.contains("F_50011")) {
            String facetVal = this.query.split("F_50011:")[1];
            if (facetVal.contains(" OR ")) {
                this.filtrableFacets = Utils.rootfacetDefIDs;
                return;
            } else if(CATEGORY_FACET_MAP.containsKey(facetVal)){
                facetVal = facetVal.split("&")[0].replaceAll("[\"()]", "");
                if(CATEGORY_FACET_MAP.containsKey(facetVal)){
                    this.filtrableFacets = ListUtils.sum(Utils.rootfacetDefIDs, CATEGORY_FACET_MAP.get(facetVal));
                    return;
                }
            } 
        }
        if(queriedFacets.contains("F_50010")){
            String facetVal = this.query.split("F_50010:")[1];
            if (facetVal.contains(" OR ")) {
                this.filtrableFacets = Utils.rootfacetDefIDs;
                return;
            } else if(CATEGORY_FACET_MAP.containsKey(facetVal)){
                facetVal = facetVal.split("&")[0].replaceAll("[\"()]", "");
                if(CATEGORY_FACET_MAP.containsKey(facetVal)){
                    this.filtrableFacets = ListUtils.sum(Utils.rootfacetDefIDs, CATEGORY_FACET_MAP.get(facetVal));
                    return;
                }
            }
        }
        this.filtrableFacets = Utils.rootfacetDefIDs;
    }

    public List<String> getFilterableFacets() {
        return this.filtrableFacets; 
    }

    public Map<String,Map<String,Integer>> removeUnwantedFacets(Map<String,Map<String,Integer>> facetMap, long numberOfResults){
        
        Set<String> facetsInQuery = new HashSet<String>(getAllMatches(this.query));
        Map<String,Map<String,Integer>> tempFacets = new TreeMap<String, Map<String,Integer>>(); 
        for(String facet : facetMap.keySet()){
            if(facetMap.get(facet).size() > 0 && !facetsInQuery.contains(facet)){
                Map<String,Integer> tempMap = new LinkedHashMap<String, Integer>();
                
                for(String facetValueName : facetMap.get(facet).keySet()){
                    //if(facetMap.get(facet).get(facetValueName) != 0 && facetMap.get(facet).get(facetValueName) != numberOfResults){
                        tempMap.put(facetValueName, facetMap.get(facet).get(facetValueName));
                    //}
                }
                if(!tempMap.isEmpty()){
                    tempFacets.put(facet, tempMap);
                }
            }   
        }
        /*if(tempFacets.containsKey("F_50010")){
            tempFacets.remove("F_50011");
        }*/

        return tempFacets;
    }
    
    public Map<String,Integer> getFacetDetails(String facetName){
        if(facetMap != null){
            return facetMap.get(facetName);
        }else{
            return null;
        }
    }
    
    public Map<String,Map<String,Integer>> getFacetMap() {
        facetMap = new TreeMap<String,Map<String,Integer>>();
        
        String facetNamePath = "/response/lst/lst[@name = 'facet_fields']/lst";
        
        NodeList nodes = null;
        try {
            nodes = (NodeList) this.xpath.evaluate(facetNamePath, this.inputSource, XPathConstants.NODESET);
        }
        catch (XPathExpressionException xpee) {
            return null;
        }
        
        if(nodes.getLength() == 0) {
            return null;
        }
        
        NodeList subNodes = null;
        
        for(int i=0; i<nodes.getLength(); i++) {
            Node node = nodes.item(i);
            String facetName = node.getAttributes().getNamedItem("name").getNodeValue();
            subNodes = node.getChildNodes();
            Map<String,Integer> facetValueCountMap = new LinkedHashMap<String,Integer>();
            for(int j=0; j<subNodes.getLength(); j++) {
                Node subNode = subNodes.item(j);
                facetValueCountMap.put(subNode.getAttributes().getNamedItem("name").getNodeValue(), Integer.parseInt(subNode.getTextContent()));
            }
            if(SORTED_FACET_VALUE_MAP.containsKey(facetName)){
                List<String> orderedValues = SORTED_FACET_VALUE_MAP.get(facetName);
                Map<String, Integer> sortedMap = new LinkedHashMap<String, Integer>();
                for (Iterator<String> it = orderedValues.iterator(); it.hasNext();) {
                    String val = it.next();
                    if(facetValueCountMap.containsKey(val)) {
                        sortedMap.put(val, facetValueCountMap.get(val));
                    }
                }
                facetMap.put(facetName, sortedMap);
            } else {
                facetMap.put(facetName, facetValueCountMap);
            }
        }
        this.numberOfResults  = this.getTotalResults();
        
        facetMap = removeUnwantedFacets(facetMap, numberOfResults);
        return facetMap;
    }
    
    public List<String> getResultMap() {
        resultMap = new LinkedList<String>();

        String resultDocsPath = "/response/result/doc";
        
        
        NodeList nodes = null;
        try {
            nodes = (NodeList) this.xpath.evaluate(resultDocsPath, this.inputSource, XPathConstants.NODESET);
        }
        catch (XPathExpressionException xpee) {
            return null;
        }
        
        if(nodes.getLength() == 0) {
            return null;
        }
        
        for(int i=0; i<nodes.getLength(); i++) {
            Node node = nodes.item(i);
            String docID = node.getFirstChild().getTextContent();
            resultMap.add(docID);   
        }
        return resultMap;
    }

    public HashMap<String, Double> getPriceStatsMap() {
        return this.getPriceStatsMap(this.inputSource);
    }
    
    public HashMap<String, Double> getPriceStatsMap(InputSource inputSource) {
        HashMap<String, Double> priceStatsMap = new HashMap<String, Double>();

        String resultDocsPath = "/response/lst[@name = 'stats']/lst[@name = 'stats_fields']/lst[@name = '" + priceFacetName + "']";
        
        
        NodeList nodes = null;
        try {
            nodes = (NodeList) this.xpath.evaluate(resultDocsPath, inputSource, XPathConstants.NODESET);
        }
        catch (XPathExpressionException xpee) {
            return null;
        }
        
        if(nodes.getLength() == 0) {
            return null;
        }
        
        NodeList subNodes = nodes.item(0).getChildNodes();
        
        for(int i=0; i<subNodes.getLength(); i++) {
            Node node = subNodes.item(i);
            
            String parameter = node.getAttributes().getNamedItem("name").getNodeValue();
            String value = node.getTextContent();
            priceStatsMap.put(parameter, Double.parseDouble(value));    
        }
        return priceStatsMap;
    }

    public HashMap<String,Integer> getRangeQueryResultMap() {
        HashMap<String, Integer> rangeQueryResultMap = new HashMap<String,Integer>();

        String resultDocsPath = "/response/lst[@name = 'facet_counts']/lst[@name = 'facet_queries']/int";
        
        
        NodeList nodes = null;
        try {
            nodes = (NodeList) this.xpath.evaluate(resultDocsPath, this.inputSource, XPathConstants.NODESET);
        }
        catch (XPathExpressionException xpee) {
            return null;
        }
        
        if(nodes.getLength() == 0) {
            return null;
        }
        
        
        for(int i=0; i<nodes.getLength(); i++) {
            Node node = nodes.item(i);

            String query = node.getAttributes().getNamedItem("name").getNodeValue();
            String docCount = node.getTextContent();

            rangeQueryResultMap.put(query,Integer.parseInt(docCount));  
        }
        return rangeQueryResultMap;
        
    }

    /**
     * 
     */
    public long getTotalResults(){
        String resultDocsPath = "/response/result";
        NodeList nodes = null;
        try {
            nodes = (NodeList) this.xpath.evaluate(resultDocsPath, this.inputSource, XPathConstants.NODESET);
        }
        catch (XPathExpressionException xpee) {
            return 0;
        }
        
        Node node = nodes.item(0);

        return Long.parseLong(node.getAttributes().getNamedItem("numFound").getNodeValue());
        
    }
        /**
     * 
     * @return
     */
    public long[] getResultEntityIDs() {
        String expression = "/response/result/doc/long";
        
        NodeList nodes = null;
        try {
            nodes = (NodeList) this.xpath.evaluate(expression, this.inputSource,
                    XPathConstants.NODESET);
        } 
        catch(XPathExpressionException xpee) {
            return null;
        }
        
        if(nodes.getLength() == 0) {
            return null;
        }
        
        long[] values = new long[nodes.getLength()];
        for(int i=0; i<nodes.getLength(); i++) {
            Node node = nodes.item(i);
            String value = node.getTextContent();
            values[i] = Long.parseLong(value);
        }
        
        return values;
    }
    
    /**
     * 
     * @return
     */
    public String[] getResultCategoryNames() {
        String expression = "/response/lst/lst[@name = 'facet_fields']/";
        expression += "lst[@name = 'Category']/int/@name";
        
        NodeList nodes = null;
        try {
            nodes = (NodeList) this.xpath.evaluate(expression, 
                this.inputSource, XPathConstants.NODESET);
        }
        catch (XPathExpressionException xpee) {
            return null;
        }
        
        if(nodes.getLength() == 0) {
            return null;
        }
        
        String[] values = new String[nodes.getLength()];
        for(int i=0; i<nodes.getLength(); i++) {
            Node node = nodes.item(i);
            values[i] = node.getTextContent();
        }
        
        return values;
    }
    
    /**
     * 
     * @return
     */
    public int[] getResultCategoryCounts() {
        String expression = "/response/lst/lst[@name = 'facet_fields']/";
        expression += "lst[@name = 'Category']/int";
        
        NodeList nodes = null;
        try {
            nodes = (NodeList) this.xpath.evaluate(expression, 
                this.inputSource, XPathConstants.NODESET);
        }
        catch (XPathExpressionException xpee) {
            return null;
        }
        
        if(nodes.getLength() == 0) {
            return null;
        }
        
        int[] values = new int[nodes.getLength()];
        for(int i=0; i<nodes.getLength(); i++) {
            Node node = nodes.item(i);
            values[i] = Integer.parseInt(node.getTextContent());
        }
        
        return values;
    }
    
    /**
     * 
     * @return
     */
    public String[]  getResultEntityNames() {
        String expression = "/response/result/doc/str";
        
        NodeList nodes = null;
        try {
            nodes = (NodeList) this.xpath.evaluate(expression, this.inputSource,
                    XPathConstants.NODESET);
        } 
        catch(XPathExpressionException xpee) {
            return null;
        }
        
        if(nodes.getLength() == 0) {
            return null;
        }
        
        String[] values = new String[nodes.getLength()];
        for(int i=0; i<nodes.getLength(); i++) {
            Node node = nodes.item(i);
            String value = node.getTextContent();
            values[i] = value;
        }
        
        return values;
    }

    /**
     * 
     * @param facetDefinitionID
     * @return
     */
    public String[] getFacetValues(String facetDefinitionID) {
        String expression = "/response/lst/lst[@name = 'facet_fields']/";
        expression += "lst[@name = '"+ facetDefinitionID +"']/int/@name";
        
        NodeList nodes = null;
        try {
            nodes = (NodeList) this.xpath.evaluate(expression, 
                this.inputSource, XPathConstants.NODESET);
        }
        catch (XPathExpressionException xpee) {
            return null;
        }
        
        if(nodes.getLength() == 0) {
            return null;
        }
        
        String[] values = new String[nodes.getLength()];
        for(int i=0; i<nodes.getLength(); i++) {
            Node node = nodes.item(i);
            values[i] = node.getTextContent();
        }
        
        return values;
    }
    
    /**
     * 
     * @param facetDefinitionID
     * @return
     */
    public String[] getFacetCounts(String facetDefinitionID) {
        String expression = "/response/lst/lst[@name = 'facet_fields']/";
        expression += "lst[@name = '" + facetDefinitionID + "']/int";
        
        NodeList nodes = null;
        try {
            nodes = (NodeList) this.xpath.evaluate(expression, 
                this.inputSource, XPathConstants.NODESET);
        }
        catch (XPathExpressionException xpee) {
            return null;
        }
        
        if(nodes.getLength() == 0) {
            return null;
        }
        
        String[] values = new String[nodes.getLength()];
        for(int i=0; i<nodes.getLength(); i++) {
            Node node = nodes.item(i);
            values[i] = node.getTextContent();
        }
        
        return values;
    }

    public static void main(String[] args){
        /*
        // Hard coded for now
        String[] facetDefIDs = new String[] {"F_50001", "F_50002", "F_50003", "F_50004", "F_50005", "F_50006", "F_50007", "F_50008", "F_50009"};

        // Hard-coded for now
        String[] facetLabels = new String[] {
            "Brand", "Price","Form Factor", "Carry In Pocket", "Cellular Technologies", 
            "Data Connectivity", "Camera Resolution", "Built-in Memory", 
            "Talk time"
        };

         */
        String[] facetDefIDs = new String[] {"Category","F_50002","F_50001",  "F_50006", "F_50007" };
        //String[] facetLabels = new String[] {"Category","Price", "Brand", "Data Connectivity", "Camera Resolution"    };

        
        String[] fqrys = {};
        SearchService search = new SearchService("nokia", fqrys, 0 , 20, null, null, null, -1);
        
        long[] entityIDs = search.getResultEntityIDs();
        log.info("entityIDs=" + Arrays.toString(entityIDs));
        
        String[] entityNames = search.getResultEntityNames();
        log.info("entityNames=" + Arrays.toString(entityNames));
        search.getFacetMap();
        
        search.getResultMap();
        search.getRangeQueryResultMap();
        search.getPriceStatsMap(new InputSource());
        search.getTotalResults();
        for (int i=0; i<facetDefIDs.length; i++) {
            search.getFacetCounts(facetDefIDs[i]);
            search.getFacetValues(facetDefIDs[i]);
        }
        
    }
    

    public static List<String> getAllMatches(String text) {
        List<String> matches = new ArrayList<String>();
        Matcher m = FACET_PATTERN.matcher(text);
        while(m.find()) {
            matches.add(m.group(1));
        }
        return matches;
    }
    
    public Map<String, Double> getDynamicPriceMap() {
        return this.dynamicPriceMap;
    }
    
}