Subversion Repositories SmartDukaan

package com.spice.profitmandi.dao.service.solr;

import com.google.gson.Gson;

import com.spice.profitmandi.common.exception.ProfitMandiBusinessException;

import com.spice.profitmandi.common.model.ProfitMandiConstants;

import com.spice.profitmandi.dao.entity.catalog.TagListing;

import com.spice.profitmandi.dao.entity.catalog.TagRanking;

import com.spice.profitmandi.dao.entity.dtr.WebProductListing;

import com.spice.profitmandi.dao.entity.similarModel.SimilarModel;

import com.spice.profitmandi.dao.repository.catalog.TagListingRepository;

import com.spice.profitmandi.dao.repository.catalog.TagRankingRepository;

import com.spice.profitmandi.dao.repository.dtr.Mongo;

import com.spice.profitmandi.dao.repository.dtr.WebListingRepository;

import com.spice.profitmandi.dao.repository.dtr.WebProductListingRepository;

import com.spice.profitmandi.dao.repository.similarModel.SimilarModelRepository;

import com.spice.profitmandi.service.tag.ItemTagModel;

import in.shop2020.model.v1.catalog.status;

import org.apache.logging.log4j.LogManager;

import org.apache.logging.log4j.Logger;

import org.apache.solr.client.solrj.SolrClient;

import org.apache.solr.client.solrj.impl.HttpSolrClient;

import org.apache.solr.common.SolrInputDocument;

import org.apache.solr.common.SolrInputField;

import org.json.JSONArray;

import org.json.JSONException;

import org.json.JSONObject;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Value;

import org.springframework.stereotype.Component;

import java.io.BufferedReader;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.net.HttpURLConnection;

import java.net.URL;

import java.net.URLConnection;

import java.time.ZoneId;

import java.util.*;

import java.util.Map.Entry;

import java.util.stream.Collectors;

import java.util.zip.GZIPInputStream;

@Component

public class FofoSolr {

    @Autowired

    private Gson gson;

    @Autowired

    private TagListingRepository tagListingRepository;

    @Autowired

    TagRankingRepository tagRankingRepository;

    @Autowired

    private Mongo contentMongoClient;

    @Autowired

    private WebListingRepository webListingRepository;

    @Autowired

    private WebProductListingRepository webProductListingRepository;

    @Value("${new.solr.url}")

    private String solrUrl;

    @Value("${reportico.url}")

    private String reporticoUrl;

    private static final Logger logger = LogManager.getLogger(FofoSolr.class);

    private static final List<Integer> CATEGORY_MASTER = Arrays.asList(10006, 10010, 14202, 14203);

    private static String getUrlContent(String urlString) {

        BufferedReader reader = null;

        try {

            URL url = new URL(urlString);

            URLConnection conn = url.openConnection();

            conn.setConnectTimeout(5000);

            conn.setReadTimeout(5000);

            reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));

            return reader.lines().collect(Collectors.joining(System.lineSeparator()));

        } catch (IOException e) {

            e.printStackTrace();

        } finally {

            if (reader != null) {

                try {

                    reader.close();

                } catch (IOException e) {

                    e.printStackTrace();

                }

            }

        }

        return null;

    }

    public JSONArray getAvailability() throws IOException, JSONException {

        String url = reporticoUrl + "?execute_mode=EXECUTE&xmlin=warehousecisnew.xml&project=FOCOR&project_password=focor&target_format=JSON";

        HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();

        connection.setRequestProperty("Accept-Encoding", "gzip");

        BufferedReader reader = null;

        try {

            InputStream inputStream = connection.getInputStream();

            if ("gzip".equals(connection.getContentEncoding())) {

                inputStream = new GZIPInputStream(inputStream);

            }

            reader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8"));

            StringBuilder responseBuilder = new StringBuilder();

            String line;

            while ((line = reader.readLine()) != null) {

                responseBuilder.append(line);

            }

            JSONObject responseJson = new JSONObject(responseBuilder.toString());

            return responseJson.getJSONArray("data");

        } finally {

            if (reader != null) {

                try {

                    reader.close();

                } catch (IOException e) {

                    // Ignore

                }

            }

        }

    }

    public JSONArray getPendingPO() throws IOException, JSONException {

        String url = reporticoUrl + "?execute_mode=EXECUTE&xmlin=UnfulfilledPOItemsNew.xml&project=FOCOR&project_password=focor&target_format=JSON";

        HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();

        connection.setRequestProperty("Accept-Encoding", "gzip");

        BufferedReader reader = null;

        try {

            InputStream inputStream = connection.getInputStream();

            if ("gzip".equals(connection.getContentEncoding())) {

                inputStream = new GZIPInputStream(inputStream);

            }

            reader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8"));

            StringBuilder responseBuilder = new StringBuilder();

            String line;

            while ((line = reader.readLine()) != null) {

                responseBuilder.append(line);

            }

            JSONObject responseJson = new JSONObject(responseBuilder.toString());

            return responseJson.getJSONArray("data");

        } finally {

            if (reader != null) {

                try {

                    reader.close();

                } catch (IOException e) {

                    // Ignore

                }

            }

        }

    }

    public Map<String, Map<String, Map<String, Integer>>> getItemMap() throws Exception {

        JSONArray availabilityJSONArray = this.getAvailability();

        JSONArray pendingPOJSONArray = this.getPendingPO();

        Map<String, Map<String, Map<String, Integer>>> availabilityItemMap = new HashMap<>();

        for (int i = 0; i < availabilityJSONArray.length(); i++) {

            JSONObject rawAvailability = availabilityJSONArray.getJSONObject(i);

            if (!availabilityItemMap.containsKey(rawAvailability.getString("Itemid"))) {

                availabilityItemMap.put(rawAvailability.getString("Itemid"),

                        new HashMap<String, Map<String, Integer>>());

            }

            if (!availabilityItemMap.get(rawAvailability.getString("Itemid"))

                    .containsKey(rawAvailability.getString("Warehouseid"))) {

                availabilityItemMap.get(rawAvailability.getString("Itemid"))

                        .put(rawAvailability.getString("Warehouseid"), new HashMap<String, Integer>());

                availabilityItemMap.get(rawAvailability.getString("Itemid"))

                        .get(rawAvailability.getString("Warehouseid")).put("netAvailability", 0);

                availabilityItemMap.get(rawAvailability.getString("Itemid"))

                        .get(rawAvailability.getString("Warehouseid")).put("netPo", 0);

            }

            int netAvailability = availabilityItemMap.get(rawAvailability.getString("Itemid"))

                    .get(rawAvailability.getString("Warehouseid")).get("netAvailability")

                    + rawAvailability.getInt("Netavailability");

            availabilityItemMap.get(rawAvailability.getString("Itemid")).get(rawAvailability.getString("Warehouseid"))

                    .put("netAvailability", netAvailability);

        }

        for (int i = 0; i < pendingPOJSONArray.length(); i++) {

            JSONObject rawPendingPo = pendingPOJSONArray.getJSONObject(i);

            String itemId = rawPendingPo.getString("Itemid");

            String warehouseId = rawPendingPo.getString("Warehouseid");

            if (!availabilityItemMap.containsKey(itemId)) {

                availabilityItemMap.put(itemId, new HashMap<String, Map<String, Integer>>());

            }

            if (!availabilityItemMap.get(itemId).containsKey(warehouseId)) {

                availabilityItemMap.get(itemId).put(warehouseId, new HashMap<String, Integer>() {

                    {

                        put("netAvailability", 0);

                        put("netPo", 0);

                    }

                });

            }

            availabilityItemMap.get(itemId).get(warehouseId).put("netPo",

                    availabilityItemMap.get(itemId).get(warehouseId).get("netPo") + rawPendingPo.getInt("Unfulfilled"));

        }

        return availabilityItemMap;

    }

    public Map<Integer, List<String>> getLabels() throws ProfitMandiBusinessException {

        List<String> labels = Arrays.asList("partner-best-sellers", "partner-price-drop", "new-launches",

                "upgrade-offer", "special-support");

        Map<Integer, String> webListing = webListingRepository.selectByUrls(labels).stream()

                .collect(Collectors.toMap(x -> x.getId(), x -> x.getUrl()));

        Map<Integer, List<WebProductListing>> webProductListingMap = webProductListingRepository

                .selectAllByWebListingIds(new ArrayList<>(webListing.keySet())).stream()

                .collect(Collectors.groupingBy(x -> x.getWebListingId()));

        Map<Integer, List<String>> catalogLabelMap = new HashMap<>();

        for (Entry<Integer, List<WebProductListing>> webProductListingEntry : webProductListingMap.entrySet()) {

            for (WebProductListing webProduct : webProductListingEntry.getValue()) {

                if (!catalogLabelMap.containsKey(webProduct.getEntityId())) {

                    catalogLabelMap.put(webProduct.getEntityId(), new ArrayList<>());

                }

                List<String> itemLabels = catalogLabelMap.get(webProduct.getEntityId());

                itemLabels.add(webListing.get(webProduct.getWebListingId()));

                catalogLabelMap.put(webProduct.getEntityId(), itemLabels);

            }

        }

        return catalogLabelMap;

    }

    @Autowired

    SimilarModelRepository similarModelRepository;

    public void populateTagItems() throws Exception {

        Map<String, Map<String, Map<String, Integer>>> availabilityItemMap = getItemMap();

        List<TagRanking> tagRankingList = tagRankingRepository.getAllTagRanking();

        List<Integer> rankingList = new ArrayList<>();

        Map<Integer, String> featureMap = new HashMap<>();

        for (TagRanking tagRanking : tagRankingList) {

            rankingList.add(tagRanking.getCatalogItemId());

            featureMap.put(tagRanking.getCatalogItemId(), tagRanking.getFeature());

        }

        //Get Similar Models

        /*List<SimilarModel> similarModels = similarModelRepository.selectAll();

        Map<Integer, List<Integer>> similarModelsMap = similarModels.stream().collect(Collectors.groupingBy(SimilarModel::getCatalogId, Collectors.mapping(x -> x.getSimilarCatalogId(), Collectors.toList())));*/

        Map<Integer, Map<String, Object>> catalogMap = new HashMap<>();

        List<status> statuses = Arrays.asList(status.ACTIVE, status.PAUSED_BY_RISK, status.PARTIALLY_ACTIVE);

        Map<Integer, List<String>> catalogLabelMap = this.getLabels();

        logger.info("catalogLabelMap {}", catalogLabelMap);

        List<ItemTagModel> itemTagModels = tagListingRepository.getAllItemTagByStatus(statuses);

        Map<String, Object> projection = new HashMap<>();

        projection.put("defaultImageUrl", 1);

        List<String> excludeBrands = Arrays.asList("Dummy", "FOC HANDSET");

        Map<Float, List<Integer>> priceModelMap = new TreeMap<>();

        for (ItemTagModel itemTagModel : itemTagModels) {

            TagListing tagListing = itemTagModel.getTagListing();

            com.spice.profitmandi.dao.entity.catalog.Item item = itemTagModel.getItem();

            if (excludeBrands.contains(item.getBrand())) {

                continue;

            }

            if (!catalogMap.containsKey(item.getCatalogItemId())) {

                if (!priceModelMap.containsKey(tagListing.getMop()) && item.getCategoryId() == 10006) {

                    priceModelMap.put(tagListing.getMop(), new ArrayList<>());

                }

                if (item.getCategoryId() == 10006) {

                    List<Integer> priceModels = priceModelMap.get(tagListing.getMop());

                    priceModels.add(item.getCatalogItemId());

                }

                Map<String, Object> catalogObj = new HashMap<>();

                catalogObj.put("stockColor", 0);

                catalogObj.put("feature", "");

                catalogObj.put("hsnCode", item.getHsnCode());

                catalogObj.put("title",

                        String.join(" ", Arrays.asList(item.getBrand(), item.getModelName(), item.getModelNumber())

                                .stream().filter(s -> s != null && !s.isEmpty()).collect(Collectors.toList())));

                catalogObj.put("brand", new String[]{item.getBrand()});

                if (item.getBrand().equals("Huawei") || item.getBrand().equals("Honor")) {

                    catalogObj.put("brand", new String[]{"Huawei", "Honor"});

                }

                if (item.getBrand().equals("Mi") || item.getBrand().equals("Xiaomi")

                        || item.getBrand().equals("Redmi")) {

                    catalogObj.put("brand", new String[]{"Mi", "Xiaomi", "Redmi"});

                }

                catalogObj.put("identifier", item.getCatalogItemId());

                catalogObj.put("items", new HashMap<Integer, Map<String, Object>>());

                Map<String, Object> filterMap = new HashMap<>();

                filterMap.put("_id", item.getCatalogItemId());

                // Don't include if catalog not available

                try {

                    String imageUrl = contentMongoClient.getEntityById(item.getCatalogItemId()).getDefaultImageUrl();

                    if (imageUrl != null) {

                        catalogObj.put("imageUrl", imageUrl);

                    } else {

                        catalogObj.put("imageUrl",

                                "https://images.smartdukaan.com/uploads/campaigns/" + item.getCatalogItemId() + ".jpg");

                    }

                } catch (Exception e) {

                    try {

                        catalogObj.put("imageUrl",

                                "https://images.smartdukaan.com/uploads/campaigns/" + item.getCatalogItemId() + ".jpg");

                    } catch (Exception e2) {

                        e2.printStackTrace();

                    }

                }

                try {

                    catalogObj.put("rank", rankingList.indexOf(item.getCatalogItemId()));

                } catch (Exception e) {

                    // A very big number

                    e.printStackTrace();

                    catalogObj.put("rank", 50000000);

                }

                if (featureMap.containsKey(item.getCatalogItemId())

                        && featureMap.get(item.getCatalogItemId()) != null) {

                    catalogObj.put("feature", featureMap.get(item.getCatalogItemId()));

                }

                catalogObj.put("categoryId", CATEGORY_MASTER.contains(item.getCategoryId()) ? item.getCategoryId() : 6);

                if (item.getCategoryId() == 10006) {

                    catalogObj.put("categoryId", 3);

                }

                catalogObj.put("subCategoryId", item.getCategoryId());

                catalogObj.put("create_timestamp",

                        tagListing.getCreatedTimestamp().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli());

                if (catalogLabelMap.containsKey(item.getCatalogItemId())

                        && catalogLabelMap.get(item.getCatalogItemId()) != null) {

                    catalogObj.put("labels", catalogLabelMap.get(item.getCatalogItemId()));

                }

                catalogMap.put(item.getCatalogItemId(), catalogObj);

                catalogObj.put("avColor", new HashMap<>());

            }

            Map<String, Object> catalogObj = catalogMap.get(item.getCatalogItemId());

            Map<Integer, Integer> avColorMap = (Map<Integer, Integer>) catalogObj.get("avColor");

            if (availabilityItemMap.containsKey(String.valueOf(item.getId()))) {

                int itemStockColor = 0;

                for (Map.Entry<String, Map<String, Integer>> entry : availabilityItemMap

                        .get(String.valueOf(item.getId())).entrySet()) {

                    String warehouseId = entry.getKey();

                    Map<String, Integer> avMap = entry.getValue();

                    if (!avColorMap.containsKey(Integer.parseInt(warehouseId))) {

                        avColorMap.put(Integer.parseInt(warehouseId), 0);

                    }

                    int color = avColorMap.get(Integer.parseInt(warehouseId));

                    if (avMap.get("netAvailability") > 0) {

                        color = 2;

                    } else if (avMap.get("netPo") > 0) {

                        color = Math.max(color, 1);

                    } else {

                        color = Math.max(color, 0);

                    }

                    avColorMap.put(Integer.parseInt(warehouseId), color);

                    catalogObj.put("avColor", avColorMap);

                    if (color > itemStockColor) {

                        itemStockColor = color;

                    }

                }

                if (itemStockColor > 0 && itemStockColor > (Integer) catalogObj.get("stockColor")) {

                    catalogObj.put("stockColor", itemStockColor);

                }

            }

            Map<Integer, Object> itemColorMap = (Map<Integer, Object>) catalogObj.get("items");

            if (!itemColorMap.containsKey(item.getId())) {

                itemColorMap.put(item.getId(), new HashMap<String, Object>() {

                    {

                        put("color", item.getColor().replace("f_", ""));

                        put("tagPricing", new ArrayList<TagListing>());

                    }

                });

            }

            Map<String, Object> itemMap = (Map<String, Object>) itemColorMap.get(item.getId());

            List<TagListing> tagPricing = (List<TagListing>) itemMap.get("tagPricing");

            tagPricing.add(tagListing);

        }

        //logger.info("catalogObj {}", catalogMap);

        List<SolrInputDocument> catalogSolrObjs = new ArrayList<>();

        for (Entry<Integer, Map<String, Object>> entry : catalogMap.entrySet()) {

            int catalogId = entry.getKey();

            Map<String, Object> catalogValMap = entry.getValue();

            Map<Integer, Object> itemsMap = (Map<Integer, Object>) catalogValMap.get("items");

            boolean active = false;

            // Map<Integer, Object> itemsMap = (Map<String, Object>)

            // catalogValMap.get("items");

            List<SolrInputDocument> itemObjs = new ArrayList<>();

            float mop = 0;

            float dp = 0;

            for (Entry<Integer, Object> itemEntry : itemsMap.entrySet()) {

                int itemId = itemEntry.getKey();

                Map<String, Object> itemMap = (Map<String, Object>) itemEntry.getValue();

                List<TagListing> tags = (List<TagListing>) itemMap.get("tagPricing");

                SolrInputDocument itemObj = new SolrInputDocument();

                for (TagListing taglist : tags) {

                    active = active || taglist.isActive();

                    itemObj.setField("id", "itemtag-" + itemId + "-" + taglist.getTagId());

                    itemObj.setField("color_s", itemMap.get("color"));

                    itemObj.setField("itemId_i", itemId);

                    itemObj.setField("tagId_i", taglist.getTagId());

                    itemObj.setField("mrp_f", taglist.getMrp());

                    itemObj.setField("mop_f", taglist.getMop());

                    itemObj.setField("sellingPrice_f", taglist.getSellingPrice());

                    itemObj.setField("active_b", taglist.isActive());

                    itemObj.setField("hot_deal_b", taglist.isHotDeals());

                    mop = taglist.getMop();

                    dp = taglist.getSellingPrice();

                }

                itemObjs.add(itemObj);

            }

            //logger.info("catalogValMap {}", catalogValMap);

            //logger.info("rank {}", catalogValMap.get("rank"));

            SolrInputDocument catalogSolrObj = new SolrInputDocument();

            catalogSolrObj.setField("id", "catalog" + catalogId);

            catalogSolrObj.setField("rank_i", catalogValMap.get("rank"));

            catalogSolrObj.setField("title_s", catalogValMap.get("title"));

            // catalogSolrObj.setField("_childDocuments_", itemObjs);

            catalogSolrObj.addChildDocuments(itemObjs);

            catalogSolrObj.setField("child_b", itemObjs.size() > 0);

            catalogSolrObj.setField("catalogId_i", catalogId);

            catalogSolrObj.setField("imageUrl_s",

                    catalogValMap.get("imageUrl").toString().replace("saholic", "smartdukaan"));

            catalogSolrObj.setField("feature_s", catalogValMap.get("feature"));

            catalogSolrObj.setField("brand_ss", catalogValMap.get("brand"));

            catalogSolrObj.setField("create_s", catalogValMap.get("create_timestamp"));

            catalogSolrObj.setField("categoryId_i", catalogValMap.get("categoryId"));

            catalogSolrObj.setField("subCategoryId_i", catalogValMap.get("subCategoryId"));

            catalogSolrObj.setField("label_ss", catalogValMap.get("labels"));

            catalogSolrObj.setField("mop_f", mop);

            catalogSolrObj.setField("dp_f", dp);

            catalogSolrObj.setField("hsncode_s", catalogValMap.get("hsnCode"));

            catalogSolrObj.setField("show_default_b", true);

            String[] brands = (String[]) catalogValMap.get("brand");

            for (String brand : brands) {

                if (brand.equals("FOC")) {

                    catalogSolrObj.setField("show_default_b", false);

                }

                if (brand.equals("Live Demo")) {

                    catalogSolrObj.setField("show_default_b", false);

                }

            }

            Map<Integer, Integer> avColorMap = (Map<Integer, Integer>) catalogValMap.get("avColor");

            for (Integer warehouseId : ProfitMandiConstants.WAREHOUSE_MAP.keySet()) {

                catalogSolrObj.setField("w" + warehouseId + "_i", 0);

            }

            for (Entry<Integer, Integer> avColorEntry : avColorMap.entrySet()) {

                int whId = avColorEntry.getKey();

                int color = avColorEntry.getValue();

                catalogSolrObj.setField("w" + whId + "_i", color);

            }

            List<Integer> similalarModels = null;

            if ((Integer) catalogValMap.get("categoryId") == 3) {

                float starPrice = mop - 500f;

                float endPrice = mop + 1000f;

                similalarModels = new ArrayList<>();

                for (Entry<Float, List<Integer>> floatListEntry : priceModelMap.entrySet()) {

                    float modelPrice = floatListEntry.getKey();

                    if (modelPrice >= starPrice && modelPrice <= endPrice) {

                        List<Integer> instock = floatListEntry.getValue().stream().filter(x->catalogMap.get(x).get("stockColor") != null && (Integer) (catalogMap.get(x).get("stockColor"))>0).collect(Collectors.toList());

                        logger.info("Entry before - {}, after - {}", floatListEntry.getValue().size(), instock.size());

                        similalarModels.addAll(instock);

                    }

                    if (modelPrice > endPrice) break;

                }

                similalarModels.remove(Integer.valueOf(catalogId));

            }

            catalogSolrObj.setField("similarModels_ii", similalarModels == null ? Collections.EMPTY_LIST : similalarModels);

            catalogSolrObj.setField("active_b", active);

            logger.info("catalogSolrObj {}", catalogSolrObj.get("rank_i"));

            catalogSolrObjs.add(catalogSolrObj);

        }

        logger.info("catalogSolrObjs {}", catalogSolrObjs);

        String solrPath = "http://" + solrUrl + ":8984/solr/demo";

        SolrClient solr = new HttpSolrClient.Builder(solrPath).build();

        solr.deleteByQuery("*:*");

        solr.add(catalogSolrObjs);

        org.apache.solr.client.solrj.response.UpdateResponse updateResponse = solr.commit();

        logger.info("solr {}", updateResponse.getStatus());

    }

    public void pushData() throws Exception {

        this.populateTagItems();

    }

}