Subversion Repositories SmartDukaan

Rev

Rev 2946 | Rev 3223 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
317 ashish 1 
/**
2 
 *
3 
 */
4 
package in.shop2020.serving.services;
5 
 
6 
 
1698 chandransh 7 
import in.shop2020.config.ConfigException;
8 
import in.shop2020.thrift.clients.config.ConfigClient;
2070 rajveer 9 
import in.shop2020.utils.CategoryManager;
1698 chandransh 10 
 
354 rajveer 11 
import java.util.Arrays;
12 
import java.util.HashMap;
790 vikas 13 
import java.util.LinkedList;
14 
import java.util.List;
354 rajveer 15 
import java.util.TreeMap;
16 
 
17 
 
317 ashish 18 
import javax.xml.xpath.XPath;
19 
import javax.xml.xpath.XPathConstants;
20 
import javax.xml.xpath.XPathExpressionException;
21 
import javax.xml.xpath.XPathFactory;
22 
 
832 rajveer 23 
import org.apache.log4j.Logger;
317 ashish 24 
import org.w3c.dom.Node;
25 
import org.w3c.dom.NodeList;
26 
import org.xml.sax.InputSource;
27 
 
354 rajveer 28 
 
317 ashish 29 
/**
545 rajveer 30 
 * @author rajveer
317 ashish 31 
 *
32 
 */
33 
public class SolrSearchService {
34 
	/**
35 
	 *
36 
	 */
2147 chandransh 37 
	private static Logger log = Logger.getLogger(Class.class);
545 rajveer 38 
 
39 
	/**
40 
	 *
41 
	 */
1698 chandransh 42 
	public static final String SOLR_URL;
317 ashish 43 
 
1698 chandransh 44 
	static {
45 
		String solr_url = null;
46 
		try {
47 
			solr_url = ConfigClient.getClient().get("solr_url");
48 
		}catch(ConfigException cex){
2949 chandransh 49 
		    log.error("Unable to get the solr URL from the config server. Setting the default value.", cex);
1698 chandransh 50 
			solr_url = "http://localhost:8983/solr/select/";
51 
		}
52 
		SOLR_URL = solr_url;
53 
	}
54 
 
317 ashish 55 
	/**
56 
	 *
57 
	 */
58 
	private XPath xpath;
59 
 
60 
	/**
61 
	 *
62 
	 */
63 
	private InputSource inputSource;
64 
 
354 rajveer 65 
	TreeMap<String,HashMap<String,Integer>> facetMap;
66 
 
790 vikas 67 
	List<String> resultMap;
545 rajveer 68 
 
69 
	long numberOfResults=0;
317 ashish 70 
	/**
71 
	 *
72 
	 * @param query
73 
	 * @param facetDefinitionIDs
74 
	 */
569 rajveer 75 
	public SolrSearchService(String query, String[] facetqueries, String[] facetDefinitionIDs, long start, long rows,  Double minPrice, Double maxPrice, long categoryId, String sortOrder) {
354 rajveer 76 
 
317 ashish 77 
		this.xpath = XPathFactory.newInstance().newXPath();
545 rajveer 78 
 
79 
    	// if multiple words are given, it should do AND for all of them
2606 rajveer 80 
		query = query.trim().replaceAll("\\s+", " ");
545 rajveer 81 
    	if(query.contains(" ")){
82 
    		String[] tokens = query.split("\\s+");
83 
    		query = tokens[0];
84 
    		for(int i = 1; i < tokens.length; i++){
85 
    			query = query + " AND " + tokens[i];
86 
    		}
87 
    	}
88 
    	log.info("query=" + query);
89 
 
317 ashish 90 
		String uri = SOLR_URL + "?wt=xml&q=" + query;
91 
 
545 rajveer 92 
		uri += "&stats=on&stats.field=F_50002";
93 
 
94 
 
95 
		if(minPrice != null || maxPrice != null){
96 
			String minString = "0";
97 
			String maxString = "*";
98 
			if(minPrice != null){
99 
				minString = minPrice.toString();
100 
			}
101 
			if(maxPrice != null){
102 
				maxString = maxPrice.toString();
103 
			}
104 
			uri += "&fq=F_50002:["+  minString + " " + maxString + "]";
354 rajveer 105 
		}
106 
 
569 rajveer 107 
		if(categoryId != 10000){
108 
			uri += "&fq=F_50010:\"" + CategoryManager.getCategoryManager().getCategoryLabel(categoryId) + "\"";
109 
		}
354 rajveer 110 
 
569 rajveer 111 
		if(sortOrder != null){
112 
			uri += "&sort=" + sortOrder;
113 
		}
545 rajveer 114 
 
317 ashish 115 
		if(facetqueries != null) {
116 
			for(int i=0; i<facetqueries.length; i++) {
536 rajveer 117 
				if(facetqueries[i].contains(" ") && !facetqueries[i].contains("F_50002")){
118 
					String[] tokens = facetqueries[i].split(":");
545 rajveer 119 
					uri += "&fq=" + tokens[0] + ":\"" + tokens[1] + "\"";
536 rajveer 120 
				}else{
121 
					uri += "&fq=" + facetqueries[i] + "";
122 
				}
317 ashish 123 
			}
124 
		}
545 rajveer 125 
		uri += "&fl=ID,Name&facet=true&start=" + start + "&rows=" + rows;
2435 rajveer 126 
		if(facetDefinitionIDs != null){
127 
			for(int i=0; i<facetDefinitionIDs.length; i++) {
128 
				uri += "&facet.field=" + facetDefinitionIDs[i];
129 
			}
317 ashish 130 
		}
2435 rajveer 131 
		log.debug("uri=" + uri);
317 ashish 132 
 
133 
		this.inputSource = new InputSource(uri);
517 rajveer 134 
 
545 rajveer 135 
		this.facetMap = getFacetMap();
354 rajveer 136 
	}
137 
 
545 rajveer 138 
	public TreeMap<String,HashMap<String,Integer>> removeUnwantedFacets(TreeMap<String,HashMap<String,Integer>> facetMap, long numberOfResults){
139 
		TreeMap<String,HashMap<String,Integer>> tempFacets = new TreeMap<String, HashMap<String,Integer>>();
354 rajveer 140 
		for(String facet : facetMap.keySet()){
545 rajveer 141 
			if(facetMap.get(facet).size() > 1){
142 
				HashMap<String,Integer> tempMap = new HashMap<String, Integer>();
143 
 
354 rajveer 144 
				for(String facetValueName : facetMap.get(facet).keySet()){
545 rajveer 145 
					if(facetMap.get(facet).get(facetValueName) != 0 && facetMap.get(facet).get(facetValueName) != numberOfResults){
146 
						tempMap.put(facetValueName, facetMap.get(facet).get(facetValueName));
354 rajveer 147 
					}
148 
				}
545 rajveer 149 
				if(!tempMap.isEmpty()){
150 
					tempFacets.put(facet, tempMap);
354 rajveer 151 
				}
545 rajveer 152 
			}
354 rajveer 153 
		}
550 rajveer 154 
		if(tempFacets.containsKey("F_50010")){
155 
			tempFacets.remove("F_50011");
156 
		}
354 rajveer 157 
 
158 
		return tempFacets;
159 
	}
160 
 
545 rajveer 161 
/*
162 
	int sumOfFacetCounts = 0;
163 
	for(String facetValueName : facetMap.get(facet).keySet()){
164 
		if(facetMap.get(facet).get(facetValueName) == 0 || facetMap.get(facet).get(facetValueName) == numberOfResults){
165 
			tempFacets.get(facet).remove(facetValueName);
166 
			// FIXME
167 
			/
168 
			HashMap<String, Integer> tmp = tempFacets.get(facet);
169 
			tmp.remove(facetValueName);
170 
			tempFacets.put(facet, tmp);
171 
			/
172 
		}else{
173 
			sumOfFacetCounts += facetMap.get(facet).get(facetValueName);
174 
		}
175 
	}
176 
	if(sumOfFacetCounts < numberOfResults*PERCENTAGE_OF_TOTAL_RESULTS/100){
177 
		tempFacets.remove(facet);
178 
	}
179 
}
180 
 
181 
*/
354 rajveer 182 
	public HashMap<String,Integer> getFacetDetails(String facetName){
2606 rajveer 183 
		if(facetMap != null){
184 
			return facetMap.get(facetName);
185 
		}else{
186 
			return null;
187 
		}
354 rajveer 188 
	}
189 
 
190 
	public TreeMap<String,HashMap<String,Integer>> getFacetMap() {
191 
		facetMap = new TreeMap<String,HashMap<String,Integer>>();
192 
 
193 
		String facetNamePath = "/response/lst/lst[@name = 'facet_fields']/lst";
545 rajveer 194 
 
354 rajveer 195 
		NodeList nodes = null;
196 
		try {
197 
			nodes = (NodeList) this.xpath.evaluate(facetNamePath, this.inputSource, XPathConstants.NODESET);
198 
		}
199 
		catch (XPathExpressionException xpee) {
200 
			return null;
201 
		}
202 
 
203 
		if(nodes.getLength() == 0) {
204 
			return null;
205 
		}
206 
 
207 
		NodeList subNodes = null;
208 
 
209 
		for(int i=0; i<nodes.getLength(); i++) {
210 
			Node node = nodes.item(i);
2946 chandransh 211 
			String facetName = node.getAttributes().getNamedItem("name").getNodeValue();
354 rajveer 212 
			subNodes = node.getChildNodes();
213 
			HashMap<String,Integer> facetValueCountMap = new HashMap<String,Integer>();
214 
			for(int j=0; j<subNodes.getLength(); j++) {
215 
				Node subNode = subNodes.item(j);
216 
				if(Integer.parseInt(subNode.getTextContent())==0)
217 
						continue;
218 
				facetValueCountMap.put(subNode.getAttributes().getNamedItem("name").getNodeValue(), Integer.parseInt(subNode.getTextContent()));
219 
			}
220 
			facetMap.put(facetName, facetValueCountMap);
221 
			}
222 
		System.out.println(facetMap);
517 rajveer 223 
 
545 rajveer 224 
		this.numberOfResults  = this.getTotalResults();
517 rajveer 225 
 
354 rajveer 226 
		facetMap = removeUnwantedFacets(facetMap, numberOfResults);
227 
		System.out.println(facetMap);
228 
		return facetMap;
229 
		}
230 
 
790 vikas 231 
	public List<String> getResultMap() {
232 
		resultMap = new LinkedList<String>();
354 rajveer 233 
 
234 
		String resultDocsPath = "/response/result/doc";
235 
 
236 
 
237 
		NodeList nodes = null;
238 
		try {
239 
			nodes = (NodeList) this.xpath.evaluate(resultDocsPath, this.inputSource, XPathConstants.NODESET);
240 
		}
241 
		catch (XPathExpressionException xpee) {
242 
			return null;
243 
		}
244 
 
245 
		if(nodes.getLength() == 0) {
246 
			return null;
247 
		}
248 
 
249 
		for(int i=0; i<nodes.getLength(); i++) {
250 
			Node node = nodes.item(i);
251 
			String docID = node.getFirstChild().getTextContent();
790 vikas 252 
			resultMap.add(docID);
354 rajveer 253 
 		}
254 
		System.out.println("resultMap is " + resultMap);
255 
		return resultMap;
256 
	}
257 
 
258 
	public HashMap<String, Double> getPriceStatsMap() {
259 
		HashMap<String, Double> priceStatsMap = new HashMap<String, Double>();
260 
 
261 
		String resultDocsPath = "/response/lst[@name = 'stats']/lst[@name = 'stats_fields']/lst[@name = 'F_50002']";
262 
 
263 
 
264 
		NodeList nodes = null;
265 
		try {
266 
			nodes = (NodeList) this.xpath.evaluate(resultDocsPath, this.inputSource, XPathConstants.NODESET);
267 
		}
268 
		catch (XPathExpressionException xpee) {
269 
			return null;
270 
		}
271 
 
272 
		if(nodes.getLength() == 0) {
273 
			return null;
274 
		}
275 
 
276 
		NodeList subNodes = nodes.item(0).getChildNodes();
277 
 
278 
		for(int i=0; i<subNodes.getLength(); i++) {
279 
			Node node = subNodes.item(i);
280 
 
281 
			String parameter = node.getAttributes().getNamedItem("name").getNodeValue();
282 
			String value = node.getTextContent();
283 
			priceStatsMap.put(parameter, Double.parseDouble(value));
284 
 		}
285 
		System.out.println("priceStatsMap is " + priceStatsMap);
286 
		return priceStatsMap;
287 
	}
288 
 
289 
	public HashMap<String,Integer> getRangeQueryResultMap() {
290 
		HashMap<String, Integer> rangeQueryResultMap = new HashMap<String,Integer>();
291 
 
292 
		String resultDocsPath = "/response/lst[@name = 'facet_counts']/lst[@name = 'facet_queries']/int";
293 
 
294 
 
295 
		NodeList nodes = null;
296 
		try {
297 
			nodes = (NodeList) this.xpath.evaluate(resultDocsPath, this.inputSource, XPathConstants.NODESET);
298 
		}
299 
		catch (XPathExpressionException xpee) {
300 
			return null;
301 
		}
302 
 
303 
		if(nodes.getLength() == 0) {
304 
			return null;
305 
		}
306 
 
307 
 
308 
		for(int i=0; i<nodes.getLength(); i++) {
309 
			Node node = nodes.item(i);
310 
 
311 
			String query = node.getAttributes().getNamedItem("name").getNodeValue();
312 
			String docCount = node.getTextContent();
313 
 
314 
			rangeQueryResultMap.put(query,Integer.parseInt(docCount));
315 
 		}
316 
		System.out.println("rangeQueryResultMap is " + rangeQueryResultMap);
317 
		return rangeQueryResultMap;
318 
 
319 
	}
320 
 
545 rajveer 321 
	/**
322 
	 *
323 
	 */
324 
	public long getTotalResults(){
325 
		String resultDocsPath = "/response/result";
326 
		NodeList nodes = null;
327 
		try {
328 
			nodes = (NodeList) this.xpath.evaluate(resultDocsPath, this.inputSource, XPathConstants.NODESET);
329 
		}
330 
		catch (XPathExpressionException xpee) {
331 
			return 0;
332 
		}
333 
 
334 
		Node node = nodes.item(0);
335 
 
336 
		return Long.parseLong(node.getAttributes().getNamedItem("numFound").getNodeValue());
337 
 
338 
	}
354 rajveer 339 
		/**
317 ashish 340 
	 *
341 
	 * @return
342 
	 */
343 
	public long[] getResultEntityIDs() {
344 
		String expression = "/response/result/doc/long";
345 
 
346 
		NodeList nodes = null;
347 
		try {
348 
			nodes = (NodeList) this.xpath.evaluate(expression, this.inputSource,
349 
					XPathConstants.NODESET);
350 
		}
351 
		catch(XPathExpressionException xpee) {
352 
			return null;
353 
		}
354 
 
355 
		if(nodes.getLength() == 0) {
356 
			return null;
357 
		}
358 
 
359 
		long[] values = new long[nodes.getLength()];
360 
		for(int i=0; i<nodes.getLength(); i++) {
361 
			Node node = nodes.item(i);
362 
			String value = node.getTextContent();
363 
			values[i] = Long.parseLong(value);
364 
 		}
365 
 
366 
		return values;
367 
	}
368 
 
369 
	/**
370 
	 *
371 
	 * @return
372 
	 */
373 
	public String[] getResultCategoryNames() {
374 
		String expression = "/response/lst/lst[@name = 'facet_fields']/";
375 
		expression += "lst[@name = 'Category']/int/@name";
376 
 
377 
		NodeList nodes = null;
378 
		try {
379 
			nodes = (NodeList) this.xpath.evaluate(expression,
380 
				this.inputSource, XPathConstants.NODESET);
381 
		}
382 
		catch (XPathExpressionException xpee) {
383 
			return null;
384 
		}
385 
 
386 
		if(nodes.getLength() == 0) {
387 
			return null;
388 
		}
389 
 
390 
		String[] values = new String[nodes.getLength()];
391 
		for(int i=0; i<nodes.getLength(); i++) {
392 
			Node node = nodes.item(i);
393 
			values[i] = node.getTextContent();
394 
 		}
395 
 
396 
		return values;
397 
	}
398 
 
399 
	/**
400 
	 *
401 
	 * @return
402 
	 */
403 
	public int[] getResultCategoryCounts() {
404 
		String expression = "/response/lst/lst[@name = 'facet_fields']/";
405 
		expression += "lst[@name = 'Category']/int";
406 
 
407 
		NodeList nodes = null;
408 
		try {
409 
			nodes = (NodeList) this.xpath.evaluate(expression,
410 
				this.inputSource, XPathConstants.NODESET);
411 
		}
412 
		catch (XPathExpressionException xpee) {
413 
			return null;
414 
		}
415 
 
416 
		if(nodes.getLength() == 0) {
417 
			return null;
418 
		}
419 
 
420 
		int[] values = new int[nodes.getLength()];
421 
		for(int i=0; i<nodes.getLength(); i++) {
422 
			Node node = nodes.item(i);
423 
			values[i] = Integer.parseInt(node.getTextContent());
424 
 		}
425 
 
426 
		return values;
427 
	}
428 
 
429 
	/**
430 
	 *
431 
	 * @return
432 
	 */
433 
	public String[]  getResultEntityNames() {
434 
		String expression = "/response/result/doc/str";
435 
 
436 
		NodeList nodes = null;
437 
		try {
438 
			nodes = (NodeList) this.xpath.evaluate(expression, this.inputSource,
439 
					XPathConstants.NODESET);
440 
		}
441 
		catch(XPathExpressionException xpee) {
442 
			return null;
443 
		}
444 
 
445 
		if(nodes.getLength() == 0) {
446 
			return null;
447 
		}
448 
 
449 
		String[] values = new String[nodes.getLength()];
450 
		for(int i=0; i<nodes.getLength(); i++) {
451 
			Node node = nodes.item(i);
452 
			String value = node.getTextContent();
453 
			values[i] = value;
454 
 		}
455 
 
456 
		return values;
457 
	}
458 
 
459 
	/**
460 
	 *
461 
	 * @param facetDefinitionID
462 
	 * @return
463 
	 */
354 rajveer 464 
	public String[] getFacetValues(String facetDefinitionID) {
317 ashish 465 
		String expression = "/response/lst/lst[@name = 'facet_fields']/";
354 rajveer 466 
		expression += "lst[@name = '"+ facetDefinitionID +"']/int/@name";
317 ashish 467 
 
468 
		NodeList nodes = null;
469 
		try {
470 
			nodes = (NodeList) this.xpath.evaluate(expression,
471 
				this.inputSource, XPathConstants.NODESET);
472 
		}
473 
		catch (XPathExpressionException xpee) {
474 
			return null;
475 
		}
476 
 
477 
		if(nodes.getLength() == 0) {
478 
			return null;
479 
		}
480 
 
481 
		String[] values = new String[nodes.getLength()];
482 
		for(int i=0; i<nodes.getLength(); i++) {
483 
			Node node = nodes.item(i);
484 
			values[i] = node.getTextContent();
545 rajveer 485 
		}
317 ashish 486 
 
487 
		return values;
488 
	}
489 
 
490 
	/**
491 
	 *
492 
	 * @param facetDefinitionID
493 
	 * @return
494 
	 */
354 rajveer 495 
	public String[] getFacetCounts(String facetDefinitionID) {
317 ashish 496 
		String expression = "/response/lst/lst[@name = 'facet_fields']/";
354 rajveer 497 
		expression += "lst[@name = '" + facetDefinitionID + "']/int";
317 ashish 498 
 
499 
		NodeList nodes = null;
500 
		try {
501 
			nodes = (NodeList) this.xpath.evaluate(expression,
502 
				this.inputSource, XPathConstants.NODESET);
503 
		}
504 
		catch (XPathExpressionException xpee) {
505 
			return null;
506 
		}
507 
 
508 
		if(nodes.getLength() == 0) {
509 
			return null;
510 
		}
511 
 
512 
		String[] values = new String[nodes.getLength()];
513 
		for(int i=0; i<nodes.getLength(); i++) {
514 
			Node node = nodes.item(i);
515 
			values[i] = node.getTextContent();
354 rajveer 516 
//			log.info("Facets Counts  " + values[i]);
317 ashish 517 
 		}
518 
 
519 
		return values;
520 
	}
545 rajveer 521 
 
522 
	public static void main(String[] args){
523 
		/*
524 
    	// Hard coded for now
525 
    	String[] facetDefIDs = new String[] {"F_50001", "F_50002", "F_50003", "F_50004", "F_50005", "F_50006", "F_50007", "F_50008", "F_50009"};
526 
 
527 
    	// Hard-coded for now
528 
    	String[] facetLabels = new String[] {
529 
	    	"Brand", "Price","Form Factor", "Carry In Pocket", "Cellular Technologies",
530 
	    	"Data Connectivity", "Camera Resolution", "Built-in Memory",
531 
	    	"Talk time"
532 
    	};
533 
 
534 
		 */
535 
    	String[] facetDefIDs = new String[] {"Category","F_50002","F_50001",  "F_50006", "F_50007" };
2147 chandransh 536 
    	//String[] facetLabels = new String[] {"Category","Price", "Brand", "Data Connectivity", "Camera Resolution"	};
545 rajveer 537 
 
538 
 
539 
    	String[] fqrys = {};
569 rajveer 540 
		SolrSearchService search = new SolrSearchService("nokia", fqrys, facetDefIDs, 0 , 20, null, null, 10000, null);
545 rajveer 541 
 
542 
    	long[] entityIDs = search.getResultEntityIDs();
543 
    	log.info("entityIDs=" + Arrays.toString(entityIDs));
544 
 
545 
    	String[] entityNames = search.getResultEntityNames();
546 
    	log.info("entityNames=" + Arrays.toString(entityNames));
547 
    	search.getFacetMap();
548 
 
549 
    	search.getResultMap();
550 
    	search.getRangeQueryResultMap();
551 
    	search.getPriceStatsMap();
552 
    	search.getTotalResults();
553 
       	for (int i=0; i<facetDefIDs.length; i++) {
554 
       		search.getFacetCounts(facetDefIDs[i]);
555 
       		search.getFacetValues(facetDefIDs[i]);
556 
       	}
557 
 
558 
	}
317 ashish 559 
}