Rev 33460 | Rev 33499 | Go to most recent revision | View as "text/plain" | Blame | Compare with Previous | Last modification | View Log | RSS feed
package com.spice.profitmandi.dao.service.solr;import com.google.gson.Gson;import com.spice.profitmandi.common.exception.ProfitMandiBusinessException;import com.spice.profitmandi.dao.entity.catalog.TagListing;import com.spice.profitmandi.dao.entity.catalog.TagRanking;import com.spice.profitmandi.dao.entity.dtr.WebProductListing;import com.spice.profitmandi.dao.repository.catalog.TagListingRepository;import com.spice.profitmandi.dao.repository.catalog.TagRankingRepository;import com.spice.profitmandi.dao.repository.dtr.Mongo;import com.spice.profitmandi.dao.repository.dtr.WebListingRepository;import com.spice.profitmandi.dao.repository.dtr.WebProductListingRepository;import com.spice.profitmandi.service.tag.ItemTagModel;import in.shop2020.model.v1.catalog.status;import org.apache.logging.log4j.LogManager;import org.apache.logging.log4j.Logger;import org.apache.solr.client.solrj.SolrClient;import org.apache.solr.client.solrj.impl.HttpSolrClient;import org.apache.solr.common.SolrInputDocument;import org.json.JSONArray;import org.json.JSONException;import org.json.JSONObject;import org.springframework.beans.factory.annotation.Autowired;import org.springframework.beans.factory.annotation.Value;import org.springframework.stereotype.Component;import java.io.BufferedReader;import java.io.IOException;import java.io.InputStream;import java.io.InputStreamReader;import java.net.HttpURLConnection;import java.net.URL;import java.net.URLConnection;import java.time.ZoneId;import java.util.*;import java.util.Map.Entry;import java.util.stream.Collectors;import java.util.zip.GZIPInputStream;@Componentpublic class FofoSolr {@Autowiredprivate Gson gson;@Autowiredprivate TagListingRepository tagListingRepository;@AutowiredTagRankingRepository tagRankingRepository;@Autowiredprivate Mongo contentMongoClient;@Autowiredprivate WebListingRepository webListingRepository;@Autowiredprivate WebProductListingRepository webProductListingRepository;@Value("${new.solr.url}")private String solrUrl;@Value("${reportico.url}")private String reporticoUrl;private static final Logger logger = LogManager.getLogger(FofoSolr.class);private static final List<Integer> CATEGORY_MASTER = Arrays.asList(10006, 10010, 14202, 14203);String solrPath = "http://" + solrUrl + ":8984/solr/demo";String mongoHost = "localhost";SolrClient solr = new HttpSolrClient.Builder(solrPath).build();private String getAvailabilityJSON() {String url = reporticoUrl + "?execute_mode=EXECUTE&xmlin=warehousecisnew.xml&project=FOCOR&project_password=focor&target_format=JSON";return getUrlContent(url);}private String getPendingPOJSON() {String url = reporticoUrl + "?execute_mode=EXECUTE&xmlin=UnfulfilledPOItemsNew.xml&project=FOCOR&project_password=focor&target_format=JSON";return getUrlContent(url);}private static String getUrlContent(String urlString) {BufferedReader reader = null;try {URL url = new URL(urlString);URLConnection conn = url.openConnection();conn.setConnectTimeout(5000);conn.setReadTimeout(5000);reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));return reader.lines().collect(Collectors.joining(System.lineSeparator()));} catch (IOException e) {e.printStackTrace();} finally {if (reader != null) {try {reader.close();} catch (IOException e) {e.printStackTrace();}}}return null;}public JSONArray getAvailability() throws IOException, JSONException {String url = reporticoUrl + "?execute_mode=EXECUTE&xmlin=warehousecisnew.xml&project=FOCOR&project_password=focor&target_format=JSON";HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();connection.setRequestProperty("Accept-Encoding", "gzip");BufferedReader reader = null;try {InputStream inputStream = connection.getInputStream();if ("gzip".equals(connection.getContentEncoding())) {inputStream = new GZIPInputStream(inputStream);}reader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8"));StringBuilder responseBuilder = new StringBuilder();String line;while ((line = reader.readLine()) != null) {responseBuilder.append(line);}JSONObject responseJson = new JSONObject(responseBuilder.toString());return responseJson.getJSONArray("data");} finally {if (reader != null) {try {reader.close();} catch (IOException e) {// Ignore}}}}public JSONArray getPendingPO() throws IOException, JSONException {String url = reporticoUrl+"?execute_mode=EXECUTE&xmlin=UnfulfilledPOItemsNew.xml&project=FOCOR&project_password=focor&target_format=JSON";HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();connection.setRequestProperty("Accept-Encoding", "gzip");BufferedReader reader = null;try {InputStream inputStream = connection.getInputStream();if ("gzip".equals(connection.getContentEncoding())) {inputStream = new GZIPInputStream(inputStream);}reader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8"));StringBuilder responseBuilder = new StringBuilder();String line;while ((line = reader.readLine()) != null) {responseBuilder.append(line);}JSONObject responseJson = new JSONObject(responseBuilder.toString());return responseJson.getJSONArray("data");} finally {if (reader != null) {try {reader.close();} catch (IOException e) {// Ignore}}}}public Map<String, Map<String, Map<String, Integer>>> getItemMap() throws Exception {JSONArray availabilityJSONArray = this.getAvailability();JSONArray pendingPOJSONArray = this.getPendingPO();Map<String, Map<String, Map<String, Integer>>> availabilityItemMap = new HashMap<>();for (int i = 0; i < availabilityJSONArray.length(); i++) {JSONObject rawAvailability = availabilityJSONArray.getJSONObject(i);if (!availabilityItemMap.containsKey(rawAvailability.getString("Itemid"))) {availabilityItemMap.put(rawAvailability.getString("Itemid"),new HashMap<String, Map<String, Integer>>());}if (!availabilityItemMap.get(rawAvailability.getString("Itemid")).containsKey(rawAvailability.getString("Warehouseid"))) {availabilityItemMap.get(rawAvailability.getString("Itemid")).put(rawAvailability.getString("Warehouseid"), new HashMap<String, Integer>());availabilityItemMap.get(rawAvailability.getString("Itemid")).get(rawAvailability.getString("Warehouseid")).put("netAvailability", 0);availabilityItemMap.get(rawAvailability.getString("Itemid")).get(rawAvailability.getString("Warehouseid")).put("netPo", 0);}int netAvailability = availabilityItemMap.get(rawAvailability.getString("Itemid")).get(rawAvailability.getString("Warehouseid")).get("netAvailability")+ rawAvailability.getInt("Netavailability");availabilityItemMap.get(rawAvailability.getString("Itemid")).get(rawAvailability.getString("Warehouseid")).put("netAvailability", netAvailability);}for (int i = 0; i < pendingPOJSONArray.length(); i++) {JSONObject rawPendingPo = pendingPOJSONArray.getJSONObject(i);String itemId = rawPendingPo.getString("Itemid");String warehouseId = rawPendingPo.getString("Warehouseid");if (!availabilityItemMap.containsKey(itemId)) {availabilityItemMap.put(itemId, new HashMap<String, Map<String, Integer>>());}if (!availabilityItemMap.get(itemId).containsKey(warehouseId)) {availabilityItemMap.get(itemId).put(warehouseId, new HashMap<String, Integer>() {{put("netAvailability", 0);put("netPo", 0);}});}availabilityItemMap.get(itemId).get(warehouseId).put("netPo",availabilityItemMap.get(itemId).get(warehouseId).get("netPo") + rawPendingPo.getInt("Unfulfilled"));}return availabilityItemMap;}public Map<Integer, List<String>> getLabels() throws ProfitMandiBusinessException {List<String> labels = Arrays.asList("partner-best-sellers", "partner-price-drop", "new-launches","upgrade-offer", "special-support");Map<Integer, String> webListing = webListingRepository.selectByUrls(labels).stream().collect(Collectors.toMap(x -> x.getId(), x -> x.getUrl()));Map<Integer, List<WebProductListing>> webProductListingMap = webProductListingRepository.selectAllByWebListingIds(new ArrayList<>(webListing.keySet())).stream().collect(Collectors.groupingBy(x -> x.getWebListingId()));Map<Integer, List<String>> catalogLabelMap = new HashMap<>();for (Entry<Integer, List<WebProductListing>> webProductListingEntry : webProductListingMap.entrySet()) {for (WebProductListing webProduct : webProductListingEntry.getValue()) {if (!catalogLabelMap.containsKey(webProduct.getEntityId())) {catalogLabelMap.put(webProduct.getEntityId(), new ArrayList<>());}List<String> itemLabels = catalogLabelMap.get(webProduct.getEntityId());itemLabels.add(webListing.get(webProduct.getWebListingId()));catalogLabelMap.put(webProduct.getEntityId(), itemLabels);}}return catalogLabelMap;}public void populateTagItems() throws Exception {Map<String, Map<String, Map<String, Integer>>> availabilityItemMap = getItemMap();List<TagRanking> tagRankingList = tagRankingRepository.getAllTagRanking();List<Integer> rankingList = new ArrayList<>();Map<Integer, String> featureMap = new HashMap<>();for (TagRanking tagRanking : tagRankingList) {rankingList.add(tagRanking.getCatalogItemId());featureMap.put(tagRanking.getCatalogItemId(), tagRanking.getFeature());}Map<Integer, Map<String, Object>> catalogMap = new HashMap<>();List<status> statuses = Arrays.asList(status.ACTIVE, status.PAUSED_BY_RISK, status.PARTIALLY_ACTIVE);Map<Integer, List<String>> catalogLabelMap = this.getLabels();logger.info("catalogLabelMap {}", catalogLabelMap);/** List<ItemTagModel> tuples =* tagListingRepository.getAllItemTagByStatus(statuses).stream() .filter(x ->* x.getItem().getCatalogItemId() == 1023747).collect(Collectors.toList());*/List<ItemTagModel> tuples = tagListingRepository.getAllItemTagByStatus(statuses);Map<String, Object> projection = new HashMap<>();projection.put("defaultImageUrl", 1);List<String> excludeBrands = Arrays.asList("Dummy", "FOC HANDSET");for (ItemTagModel result : tuples) {TagListing tag = result.getTagListing();com.spice.profitmandi.dao.entity.catalog.Item item = result.getItem();if (excludeBrands.contains(item.getBrand())) {continue;}if (!catalogMap.containsKey(item.getCatalogItemId())) {Map<String, Object> catalogObj = new HashMap<>();catalogObj.put("feature", "");catalogObj.put("hsnCode", item.getHsnCode());catalogObj.put("title",String.join(" ", Arrays.asList(item.getBrand(), item.getModelName(), item.getModelNumber()).stream().filter(s -> s != null && !s.isEmpty()).collect(Collectors.toList())));catalogObj.put("brand", new String[] { item.getBrand() });if (item.getBrand().equals("Huawei") || item.getBrand().equals("Honor")) {catalogObj.put("brand", new String[] { "Huawei", "Honor" });}if (item.getBrand().equals("Mi") || item.getBrand().equals("Xiaomi")|| item.getBrand().equals("Redmi")) {catalogObj.put("brand", new String[] { "Mi", "Xiaomi", "Redmi" });}catalogObj.put("identifier", item.getCatalogItemId());catalogObj.put("items", new HashMap<Integer, Map<String, Object>>());Map<String, Object> filterMap = new HashMap<>();filterMap.put("_id", item.getCatalogItemId());// Don't include if catalog not availabletry {String imageUrl = contentMongoClient.getEntityById(item.getCatalogItemId()).getDefaultImageUrl();if (imageUrl != null) {catalogObj.put("imageUrl", imageUrl);} else {catalogObj.put("imageUrl","https://images.smartdukaan.com/uploads/campaigns/" + item.getCatalogItemId() + ".jpg");}} catch (Exception e) {try {catalogObj.put("imageUrl","https://images.smartdukaan.com/uploads/campaigns/" + item.getCatalogItemId() + ".jpg");} catch (Exception e2) {e2.printStackTrace();}}try {catalogObj.put("rank", rankingList.indexOf(item.getCatalogItemId()));} catch (Exception e) {// A very big numbere.printStackTrace();catalogObj.put("rank", 50000000);}if (featureMap.containsKey(item.getCatalogItemId())&& featureMap.get(item.getCatalogItemId()) != null) {catalogObj.put("feature", featureMap.get(item.getCatalogItemId()));}catalogObj.put("categoryId", CATEGORY_MASTER.contains(item.getCategoryId()) ? item.getCategoryId() : 6);if (item.getCategoryId() == 10006) {catalogObj.put("categoryId", 3);}catalogObj.put("subCategoryId", item.getCategoryId());catalogObj.put("create_timestamp",tag.getCreatedTimestamp().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli());if (catalogLabelMap.containsKey(item.getCatalogItemId())&& catalogLabelMap.get(item.getCatalogItemId()) != null) {catalogObj.put("labels", catalogLabelMap.get(item.getCatalogItemId()));}catalogMap.put(item.getCatalogItemId(), catalogObj);catalogObj.put("avColor", new HashMap<>());}Map<String, Object> catalogObj = catalogMap.get(item.getCatalogItemId());Map<Integer, Integer> avColorMap = (Map<Integer, Integer>) catalogObj.get("avColor");if (availabilityItemMap.containsKey(String.valueOf(item.getId()))) {for (Map.Entry<String, Map<String, Integer>> entry : availabilityItemMap.get(String.valueOf(item.getId())).entrySet()) {String warehouseId = entry.getKey();Map<String, Integer> avMap = entry.getValue();if (!avColorMap.containsKey(Integer.parseInt(warehouseId))) {avColorMap.put(Integer.parseInt(warehouseId), 0);}int color = avColorMap.get(Integer.parseInt(warehouseId));if (avMap.get("netAvailability") > 0) {color = 2;} else if (avMap.get("netAvailability") + avMap.get("netPo") > 0) {color = Math.max(color, 1);} else {color = Math.max(color, 0);}avColorMap.put(Integer.parseInt(warehouseId), color);catalogObj.put("avColor", avColorMap);}}Map<Integer, Object> itemColorMap = (Map<Integer, Object>) catalogObj.get("items");if (!itemColorMap.containsKey(item.getId())) {itemColorMap.put(item.getId(), new HashMap<String, Object>() {{put("color", item.getColor().replace("f_", ""));put("tagPricing", new ArrayList<TagListing>());}});}Map<String, Object> itemMap = (Map<String, Object>) itemColorMap.get(item.getId());List<TagListing> tagPricing = (List<TagListing>) itemMap.get("tagPricing");tagPricing.add(tag);}logger.info("catalogObj {}", catalogMap);List<SolrInputDocument> catalogSolrObjs = new ArrayList<>();for (Entry<Integer, Map<String, Object>> entry : catalogMap.entrySet()) {int catalogId = entry.getKey();Map<String, Object> catalogValMap = entry.getValue();Map<Integer, Object> itemsMap = (Map<Integer, Object>) catalogValMap.get("items");boolean active = false;// Map<Integer, Object> itemsMap = (Map<String, Object>)// catalogValMap.get("items");List<SolrInputDocument> itemObjs = new ArrayList<>();float mop = 0;for (Entry<Integer, Object> itemEntry : itemsMap.entrySet()) {int itemId = itemEntry.getKey();Map<String, Object> itemMap = (Map<String, Object>) itemEntry.getValue();List<TagListing> tags = (List<TagListing>) itemMap.get("tagPricing");SolrInputDocument itemObj = new SolrInputDocument();for (TagListing taglist : tags) {active = active || (Boolean) taglist.isActive();itemObj.setField("id", "itemtag-" + itemId + "-" + taglist.getTagId());itemObj.setField("color_s", itemMap.get("color"));itemObj.setField("itemId_i", itemId);itemObj.setField("tagId_i", taglist.getTagId());itemObj.setField("mrp_f", taglist.getMrp());itemObj.setField("mop_f", taglist.getMop());itemObj.setField("sellingPrice_f", taglist.getSellingPrice());itemObj.setField("active_b", taglist.isActive());itemObj.setField("hot_deal_b", taglist.isHotDeals());mop = taglist.getMop();}itemObjs.add(itemObj);}logger.info("catalogValMap {}", catalogValMap);logger.info("rank {}", catalogValMap.get("rank"));SolrInputDocument catalogSolrObj = new SolrInputDocument();catalogSolrObj.setField("id", "catalog" + catalogId);catalogSolrObj.setField("rank_i", catalogValMap.get("rank"));catalogSolrObj.setField("title_s", catalogValMap.get("title"));// catalogSolrObj.setField("_childDocuments_", itemObjs);catalogSolrObj.addChildDocuments(itemObjs);catalogSolrObj.setField("child_b", itemObjs.size() > 0);catalogSolrObj.setField("catalogId_i", catalogId);catalogSolrObj.setField("imageUrl_s",catalogValMap.get("imageUrl").toString().replace("saholic", "smartdukaan"));catalogSolrObj.setField("feature_s", catalogValMap.get("feature"));catalogSolrObj.setField("brand_ss", catalogValMap.get("brand"));catalogSolrObj.setField("create_s", catalogValMap.get("create_timestamp"));catalogSolrObj.setField("categoryId_i", catalogValMap.get("categoryId"));catalogSolrObj.setField("subCategoryId_i", catalogValMap.get("subCategoryId"));catalogSolrObj.setField("label_ss", catalogValMap.get("labels"));catalogSolrObj.setField("w7573_i", 0);catalogSolrObj.setField("w7678_i", 0);catalogSolrObj.setField("w7720_i", 0);catalogSolrObj.setField("w8468_i", 0);catalogSolrObj.setField("w8889_i", 0);catalogSolrObj.setField("w8947_i", 0);catalogSolrObj.setField("w9213_i", 0);catalogSolrObj.setField("w9203_i", 0);catalogSolrObj.setField("mop_f", mop);catalogSolrObj.setField("hsncode_s", catalogValMap.get("hsnCode"));catalogSolrObj.setField("show_default_b", true);String[] brands = (String[]) catalogValMap.get("brand");for (String brand : brands) {if (brand.equals("FOC")) {catalogSolrObj.setField("show_default_b", false);}if (brand.equals("Live Demo")) {catalogSolrObj.setField("show_default_b", false);}}Map<Integer, Integer> avColorMap = (Map<Integer, Integer>) catalogValMap.get("avColor");for (Entry<Integer, Integer> avColorEntry : avColorMap.entrySet()) {int whId = avColorEntry.getKey();int color = (int) avColorEntry.getValue();catalogSolrObj.setField("w" + whId + "_i", color);}catalogSolrObj.setField("active_b", active);logger.info("catalogSolrObj {}", catalogSolrObj.get("rank_i"));catalogSolrObjs.add(catalogSolrObj);}logger.info("catalogSolrObjs {}", catalogSolrObjs);solr.deleteByQuery("*:*");solr.add(catalogSolrObjs);org.apache.solr.client.solrj.response.UpdateResponse updateResponse = solr.commit();logger.info("solr {}", updateResponse.getStatus());}public void pushData() throws Exception {this.populateTagItems();}}