Rev 8780 | Blame | Compare with Previous | Last modification | View Log | RSS feed
package in.shop2020.util;import in.shop2020.metamodel.core.Entity;import in.shop2020.metamodel.core.Feature;import in.shop2020.metamodel.core.Media;import in.shop2020.metamodel.core.Slide;import in.shop2020.metamodel.definitions.Catalog;import in.shop2020.metamodel.definitions.Category;import in.shop2020.metamodel.util.CreationUtils;import in.shop2020.metamodel.util.ExpandedBullet;import in.shop2020.metamodel.util.ExpandedEntity;import in.shop2020.metamodel.util.ExpandedFeature;import in.shop2020.model.v1.catalog.CatalogService.Client;import in.shop2020.model.v1.catalog.Item;import in.shop2020.model.v1.catalog.status;import in.shop2020.model.v1.user.ItemCouponDiscount;import in.shop2020.model.v1.user.PromotionException;import in.shop2020.model.v1.user.PromotionService;import in.shop2020.thrift.clients.CatalogClient;import in.shop2020.thrift.clients.PromotionClient;import in.shop2020.thrift.clients.WarehouseClient;import in.shop2020.thrift.clients.config.ConfigClient;import in.shop2020.ui.util.NewVUI;import in.shop2020.utils.ConfigClientKeys;import in.shop2020.utils.GmailUtils;import in.shop2020.utils.Logger;import in.shop2020.warehouse.WarehouseService;import java.io.BufferedInputStream;import java.io.File;import java.io.FileInputStream;import java.io.FileNotFoundException;import java.io.FileOutputStream;import java.io.FileReader;import java.io.FileWriter;import java.io.IOException;import java.io.InputStream;import java.io.ObjectInput;import java.io.ObjectInputStream;import java.io.ObjectOutputStream;import java.io.Reader;import java.text.MessageFormat;import java.text.SimpleDateFormat;import java.util.ArrayList;import java.util.Arrays;import java.util.Calendar;import java.util.Date;import java.util.HashMap;import java.util.HashSet;import java.util.Iterator;import java.util.List;import java.util.Map;import java.util.Set;import java.util.regex.Matcher;import java.util.regex.Pattern;import javax.mail.MessagingException;import org.apache.commons.collections.CollectionUtils;import org.apache.commons.collections.Predicate;import org.apache.commons.lang.StringUtils;import org.apache.commons.logging.Log;import org.apache.commons.logging.LogFactory;import org.apache.commons.vfs.FileObject;import org.apache.commons.vfs.FileSystemException;import org.apache.commons.vfs.FileSystemOptions;import org.apache.commons.vfs.Selectors;import org.apache.commons.vfs.impl.StandardFileSystemManager;import org.apache.commons.vfs.provider.sftp.SftpFileSystemConfigBuilder;import org.apache.thrift.TException;import org.apache.thrift.transport.TTransportException;import com.google.gson.Gson;import com.google.gson.reflect.TypeToken;/**** @author amit** Utity class to generate a TSV file to be uploaded to Amazon's seller* central. ItemNumbers.xls which is fetched from Nilesh to get UPC/EAN.* Accesses Catalog database to pull Mobile related entities and* CMS(bdb) to pull the data to generate the tsv file.*/@SuppressWarnings("unchecked")public class AmazonSCDataGenerator1 {private static final Map<Long, Long> dupItemsMap = new HashMap<Long, Long>();private static final Map<Long, String> dupItemNumbersMap = new HashMap<Long, String>();public static final int ITEM_NUMBER_MAP = 0;public static final int ITEM_NUMBER_TYPE_MAP = 1;public static final String FTP_UPLOAD_URL = "dar-eu.amazon-digital-ftp.com";public static final String FTP_USER_NAME = "M_SAHOLIC_107503884";public static final int FTP_PORT = 22;public static final String FTP_PASSWORD = "S2rT2qvKLG";public static final List<Long> PAUSED_BY_RISK_ITEMS = getPausedByRiskItems();Pattern patt = Pattern.compile("(\\d*\\.?\\d+)");// private static DecimalFormat numberFormat = new DecimalFormat("#");private static SimpleDateFormat dateformatYYYYMMDDHHMM = new SimpleDateFormat("yyyyMMdd-kkmm");private static Predicate predicateActivePaused;private static Predicate predicateLaptopMobiles;private static Map<Long, Map<String, Object>> itemMap;private static Map<Long, Map<String, Object>> itemMapOld;private static boolean isFirstPass = true;private static String objFile = "itemMap.obj";public static StringBuffer mailAlerMessage = new StringBuffer();private static Log log = LogFactory.getLog(AmazonSCDataGenerator1.class);static {dupItemsMap.put(5l, 999999999l);dupItemsMap.put(4859l, 999999998l);dupItemsMap.put(2039l, 999999997l);/* dupItemsMap.put(3880l, 999999996l); */dupItemsMap.put(2082l, 999999995l);dupItemsMap.put(2215l, 999999994l);dupItemsMap.put(2144l, 999999983l);predicateLaptopMobiles = new Predicate() {@Overridepublic boolean evaluate(Object obj) {Item item = (Item) obj;if (item.getCategory() == 0)return false;return (Catalog.getInstance().getDefinitionsContainer().getCategory(item.getCategory()).getParentCategory().getID() == 10001|| Catalog.getInstance().getDefinitionsContainer().getCategory(item.getCategory()).getParentCategory().getID() == 10011|| Catalog.getInstance().getDefinitionsContainer().getCategory(item.getCategory()).getID() == 10050 || Catalog.getInstance().getDefinitionsContainer().getCategory(item.getCategory()).getID() == 10010);}};predicateActivePaused = new Predicate() {@Overridepublic boolean evaluate(Object obj) {Item item = (Item) obj;return Arrays.asList(new status[] { status.ACTIVE, status.PAUSED,status.PAUSED_BY_RISK, status.COMING_SOON }).contains(item.getItemStatus());}};/** //try loading promotion client try { pc = new* PromotionClient().getClient(); } catch(TTransportException e){* System.out.println("Error accessing promotion Service: " + e); }*/}private PromotionService.Client pc = null;private WarehouseService.Client whc = null;private Client cc = null;private Client ccProd = null;private FileWriter productFileWriter;private FileWriter cameraFileWriter;private FileWriter laptopFileWriter;private List<Entity> validEntites;private String generationType;String dateString = dateformatYYYYMMDDHHMM.format(new Date());public AmazonSCDataGenerator1(List<Entity> validEntities, String type) {try {pc = new PromotionClient().getClient();} catch (TTransportException e) {System.out.println("Error accessing promotion Service: " + e);}try {whc = new WarehouseClient().getClient();} catch (TTransportException e) {System.out.println("Error accessing warehouse Service: " + e);}try {cc = new CatalogClient().getClient();} catch (TTransportException e) {System.out.println("Error accessing warehouse Service: " + e);}try {ccProd = new CatalogClient(ConfigClientKeys.catalog_service_server_host_prod.toString(), ConfigClientKeys.catalog_service_server_port.toString()).getClient();} catch (TTransportException e) {System.out.println("Error accessing warehouse Service: " + e);}File laptopsFile = new File(Utils.EXPORT_PATH + "Laptops" + type + dateString + ".tsv");try {laptopFileWriter = new FileWriter(laptopsFile, true);} catch (IOException e) {// TODO Auto-generated catch blocke.printStackTrace();}File productsFile = new File(Utils.EXPORT_PATH + "Products" + type + dateString + ".tsv");try {productFileWriter = new FileWriter(productsFile, true);} catch (IOException e) {// TODO Auto-generated catch blocke.printStackTrace();}File camerasFile = new File(Utils.EXPORT_PATH + "Cameras" + type + dateString + ".tsv");try {cameraFileWriter = new FileWriter(camerasFile, true);} catch (IOException e) {// TODO Auto-generated catch blocke.printStackTrace();}this.validEntites = validEntities;this.generationType = type;}/*** @param args*/public static void main(String[] args) throws Exception {AmazonSCDataGenerator1.generatePricesAndAvailability(Boolean.parseBoolean(args[0]));}private static String normalizeBatteryType(String batteryType) {if (batteryType.contains("Li-Ion") || batteryType.contains("Li-ion")|| batteryType.contains("Lithium-ion")) {batteryType = "lithium_ion";} else if (batteryType.contains("Li-Po")) {batteryType = "lithium_metal";} else {batteryType = "";}return batteryType;}private static String normalizeFormFactor(String formFactor) {if (formFactor.equals("Candybar")) {formFactor = "candy-bar";} else if (formFactor.equals("Slider")) {formFactor = "slide";} else if (formFactor.equals("Flip") || formFactor.equals("Clamshell")) {formFactor = "slide";} else {formFactor = "";}return formFactor;}private static String normalizeOperatingSystem(String opsys) {if (opsys.contains("Android")) {opsys = "Android";} else if (opsys.contains("Symbian")) {opsys = "Symbian";} else if (opsys.contains("BlackBerry")) {opsys = "Blackberry";} else if (opsys.contains("Windows")) {opsys = "Windows Phone";} else if (opsys.contains("bada") || opsys.contains("Bada")) {opsys = "Bada";} else if (opsys.contains("iOS")) {opsys = "iOS";} else {opsys = "";}return opsys;}/**** @return the complete URL of a phone.*/private static String getUrl(Item item, String type) {String url = "http://www.saholic.com/" + type + "/";String productUrl = getHyphenatdName(item);productUrl = productUrl + "-" + item.getCatalogItemId();url = url + productUrl;url = url.replace("--", "-");return url;}/**** @param item* @return the URL path for a mobile phone.*/private static String getHyphenatdName(Item item) {String productUrl = item.getBrand();if (item.getModelName() != null) {productUrl = productUrl + "-" + item.getModelName();}if (item.getModelNumber() != null) {productUrl = productUrl + '-' + item.getModelNumber();}productUrl = productUrl.replace("/", "-");productUrl = productUrl.replace(" ", "-");productUrl = productUrl.replace("--", "-");productUrl = productUrl.toLowerCase();return productUrl;}public static Map<Long, String> extractFeatureValuesForEntity(List<Long> featureIds, Entity entity) throws Exception {Map<Long, String> featureIdValuesMap = new HashMap<Long, String>();for (Slide slide : entity.getSlides()) {if (slide.getSlideDefinitionID() == 130001L) {featureIdValuesMap.put(AmazonFeatures.DESCRIPTION.getValue(),slide.getFreeformContent().getFreeformText().trim().replaceAll("\\s+", " "));}if (slide.hasChildrenSlides()) {for (Slide sl : slide.getChildrenSlides()) {if (sl.getFeatures() != null) {slide.getFeatures().addAll(sl.getFeatures());}if (sl.hasChildrenSlides()) {for (Slide sl2 : sl.getChildrenSlides()) {if (sl2.getFeatures() != null) {slide.getFeatures().addAll(sl2.getFeatures());}}}}}if (slide.getFeatures() == null) {continue;}for (Feature feature : slide.getFeatures()) {if (featureIds.contains(feature.getFeatureDefinitionID())) {String value = "";ExpandedFeature expFeature = new ExpandedFeature(feature);if (expFeature.getExpandedBullets() != null) {int count = 0;for (ExpandedBullet expBullet : expFeature.getExpandedBullets()) {if (count == 0) {value = value + expBullet.displayText();} else {value = value + " -- "+ expBullet.displayText();}count++;}}featureIdValuesMap.put(feature.getFeatureDefinitionID(),value);}}}return featureIdValuesMap;}public static Map<Long, String> extractFeatureValuesForEntity(Entity entity)throws Exception {Map<Long, String> featureIdValuesMap = new HashMap<Long, String>();for (Slide slide : entity.getSlides()) {if (slide.getSlideDefinitionID() == 130001L) {featureIdValuesMap.put(AmazonFeatures.DESCRIPTION.getValue(),slide.getFreeformContent().getFreeformText().trim().replaceAll("\\s+", " "));}if (slide.hasChildrenSlides()) {for (Slide sl : slide.getChildrenSlides()) {if (sl.getFeatures() != null) {slide.getFeatures().addAll(sl.getFeatures());}if (sl.hasChildrenSlides()) {for (Slide sl2 : sl.getChildrenSlides()) {if (sl2.getFeatures() != null) {slide.getFeatures().addAll(sl2.getFeatures());}}}}}if (slide.getFeatures() == null) {continue;}for (Feature feature : slide.getFeatures()) {String value = "";ExpandedFeature expFeature = new ExpandedFeature(feature);if (expFeature.getExpandedBullets() != null) {int count = 0;for (ExpandedBullet expBullet : expFeature.getExpandedBullets()) {if (count == 0) {value = value + expBullet.displayText();} else {value = value + " -- " + expBullet.displayText();}count++;}}featureIdValuesMap.put(feature.getFeatureDefinitionID(), value);}}return featureIdValuesMap;}public static Map<Long, String> getAllImageUrls(Entity entity, List<Item> items)throws Exception {Map<Long, String> imageUrlMap = new HashMap<Long, String>();ExpandedEntity expEntity = new ExpandedEntity(entity);Map<String, Media> medias = expEntity.getAllMedias();for (Item item : items) {String url = "";String color = item.getColor();if (color == null) {color = "jsfhdjsfdhjfdfhdjffh";}String parts[] = color.split("(-|&|\\s|\\+|and|NA|\\.)+");url = "http://static0.saholic.com/images/media"+ File.separator+ entity.getID()+ File.separator+ NewVUI.computeNewFileName(EntityUtils.getMediaPrefix(expEntity), "default.jpg", String.valueOf(EntityUtils.getCreationTimeFromSummarySlide(expEntity,"default")));for (String part : parts) {if (part.length() < 3) {continue;}for (String label : expEntity.getAllImageLabels()) {if (!StringUtils.containsIgnoreCase(label, part.trim())) {url = "http://static0.saholic.com/images/media"+ File.separator+ entity.getID()+ File.separator+ NewVUI.computeNewFileName(EntityUtils.getMediaPrefix(expEntity),"default.jpg",String.valueOf(EntityUtils.getCreationTimeFromSummarySlide(expEntity, "default")));continue;} else {url = "http://static0.saholic.com/images/media"+ File.separator+ entity.getID()+ File.separator+ NewVUI.computeNewFileName(EntityUtils.getMediaPrefix(expEntity),medias.get(label).getFileName(),String.valueOf(medias.get(label).getCreationTime().getTime()));break;}}}imageUrlMap.put(item.getId(), url);}return imageUrlMap;}/**** @return the title of the Item in the format required by Amazon: <Brand>* <Model Name> <Model Number> | <Color>*/private static String getTitle(Item item) {String title = item.getBrand();if (!StringUtils.isEmpty(item.getModelName())) {title = title + " " + item.getModelName();}if (!StringUtils.isEmpty(item.getModelNumber())) {title = title + " " + item.getModelNumber();}if (!(StringUtils.isEmpty(item.getColor()) || item.getColor().equalsIgnoreCase("NA"))) {title = title + " | " + item.getColor();}return title;}private static String normalizeScreenType(String screenType) {if (screenType == null) {return "";}if (screenType.indexOf("LCD") >= 0 || screenType.indexOf("Nova") >= 0|| screenType.indexOf("Retina") >= 0) {screenType = "LCD";} else if (screenType.indexOf("LED") >= 0) {screenType = "LED";} else {screenType = "";}return screenType;}private static String getCellularTechnology(String multipleSIM,String network3G) {String cellularTechnology = "";if ("Dual-SIM".equals(multipleSIM)) {cellularTechnology = "Dual SIM";}if (StringUtils.isEmpty(cellularTechnology)) {if (!"".equals(network3G)) {cellularTechnology = "3G";} else {cellularTechnology = "GSM";}}return cellularTechnology;}private void generateLaptopData(List<Entity> entities,String generationType) throws Exception {StringBuffer tsvBuffer = new StringBuffer();if (isFirstPass) {tsvBuffer.append(StringUtils.join(new String[] {"Amazon.com Product Ads Header","Purge-Replace=false", "", "", "", "", "", "", "","", "", "", "", "", "", "", "", "", "", "", "", "","", "", "", "", "", "", "", "", "", "", "", "", "","", "", "", "", "", "", "", "", "", "", "", "", "","", "", "", "", "", "", "", "", "", "", "", "", "","", "", "", "" }, "\t")).append("\n");tsvBuffer.append(StringUtils.join(new String[] { "SKU", "Title", "Link","Price", "Delivery Time","Recommended Browse Node", "Standard Product ID","Product ID Type", "Category", "Brand","Manufacturer", "Mfr part number", "Model Number","Description", "Shipping Cost", "Image","List Price", "Availability", "Operating System","Processor Type", "Processor Brand","Computer CPU speed", "Hard disk size","Included RAM size", "Display size","Display Technology", "Memory Storage Capacity","Connectivity Technology", "Wireless Type","Screen Resolution", "Memory Card Type","Colour Name", "Colour Map","Item package quantity", "Warranty","Assembly required", "Battery Cell Composition","Batteries Included", "Batteries Required","Power Source", "Power Adapter Included","Shipping Weight", "Weight", "Length", "Height","Width", "Keywords1", "Keywords2", "Keywords3","Keywords4", "Keywords5", "Bullet point1","Bullet point2", "Bullet point3", "Bullet point4","Bullet point5", "Other image-url1","Other image-url2", "Other image-url3","Other image-url4", "Other image-url5","Offer note", "Discount Amount", "Junglee Exclusive Offer", "Offer Promo Code", "Is Gift Wrap Available","Registered Parameter", "Update Delete", "Hard Disk Format", "Accepted Payment Methods" }, "\t")).append("\n");laptopFileWriter.write(tsvBuffer.toString());laptopFileWriter.flush();tsvBuffer.setLength(0);}for (Entity entity : entities) {if (Catalog.getInstance().getDefinitionsContainer().getCategory(entity.getCategoryID()) == null)continue;long categoryId = Catalog.getInstance().getDefinitionsContainer().getCategory(entity.getCategoryID()).getID();if (entity == null || entity.getSlides() == null|| (categoryId != 10050 && categoryId != 10010 && categoryId != 10073) ) {continue;}Map<Long, String> fvalues = extractFeatureValuesForEntity(entity);String displayTechnology = StringUtils.defaultString(fvalues.get(120008L));if(displayTechnology.equals("")){displayTechnology = StringUtils.defaultString(fvalues.get(120175L));}List<Item> items = getItemsByCatalogId(entity.getID());Map<Long, String> itemIdImageUrlMap = getAllImageUrls(entity, items);CollectionUtils.filter(items, predicateActivePaused);if (CollectionUtils.isEmpty(items))continue;for (Item item : items) {String productId = "";String productType = "";Long itemId = item.getId();populateItemMap(item);List<String> itemNumbers = getItemNumbers(itemId);if (!itemNumbers.isEmpty()) {for (String itemNumber : itemNumbers) {itemNumber = itemNumber.replaceAll("-", "");try {Long.parseLong(itemNumber);} catch (NumberFormatException nfe) {continue;}if (itemNumber.length() == 13) {productType = "EAN";} else if (itemNumber.length() == 12) {productType = "UPC";}productId = itemNumber;break;}}if (item.getColor() == null) {item.setColor("");}String imageUrl = itemIdImageUrlMap.get(itemId);if (categoryId != 10073) {tsvBuffer.append(StringUtils.join(new String[] {itemId.toString(),getTitle(item),getUrl(item,categoryId == 10010 ? "tablets": "laptops"),"" + (int) (item.getSellingPrice()),"1",categoryId == 10010 ? "803574031": "803540031",productId,// productId,productType, // productType,categoryId == 10010 ? "Tablet": "Laptop",item.getBrand(),item.getBrand(),"",item.getModelNumber(),StringUtils.defaultString(fvalues.get(AmazonFeatures.DESCRIPTION.getValue())),"0",imageUrl,""+ (item.getMrp() == 0 ? (int) item.getSellingPrice(): (int) (item.getMrp())),getAvailability(item),getOS(StringUtils.defaultString(fvalues.get(120197L))), // Operating// SystemgetType(StringUtils.defaultString(fvalues.get(120160L))), // Processor// TypeStringUtils.defaultString(fvalues.get(120160L)).split(" ")[0],StringUtils.defaultString(fvalues.get(120170L)),// Computer// CPU// SpeedStringUtils.defaultString(fvalues.get(120112L)),// HD SizeStringUtils.defaultString(fvalues.get(120031L)),// RAMgetScreenSize(StringUtils.defaultString(fvalues.get(120005L))),// Screen// SizedisplayTechnology,// Display// technologyStringUtils.defaultString(fvalues.get(120112L)),// Memory// Storage// Capacity"",// Connectivity TechnologygetWirelessType(StringUtils.defaultString(fvalues.get(120178L))), // Wireless// TypegetScreenResolution(StringUtils.defaultString(fvalues.get(categoryId == 10010 ? 120006L: 120174L))),// Screen// Resolution"",item.getColor(),"","","","","","","","","","","","","","","","","","","","","","","","","","","","","",getOfferText(item),"","",item.getItemStatus().equals(status.PHASED_OUT) ? "Delete": "","","credit_card|debit_card|net_banking|emi|prepaid_card"+ (Double.valueOf(250D).compareTo(item.getSellingPrice()) <= 0&& Double.valueOf(25000D).compareTo(item.getSellingPrice()) >= 0 ? "|cash_on_delivery": "") }, "\t")).append("\n");} else {String hardDiskFormat = StringUtils.defaultString(fvalues.get(120284l)).split(" ")[0];tsvBuffer.append(StringUtils.join(new String[] {itemId.toString(),getTitle(item),getUrl(item, "laptop-accessories"),"" + (int) (item.getSellingPrice()),"1","803511031",productId,// productId,productType, // productType,"External Hard Drives",item.getBrand(),item.getBrand(),"",item.getModelNumber(),StringUtils.defaultString(fvalues.get(AmazonFeatures.DESCRIPTION.getValue())),"0",imageUrl,""+ (item.getMrp() == 0 ? (int) item.getSellingPrice(): (int) (item.getMrp())),getAvailability(item),"",// Operating System"", // Processor Type"", // Processor Brand"",// Computer CPU SpeedStringUtils.defaultString(fvalues.get(120112L)),// HD Size"",// RAM"",// Screen Size"",// Display technologyStringUtils.defaultString(fvalues.get(120112L)),// Memory// Storage// Capacity"",// Connectivity Technology"", // Wireless Type"",// Screen Resolution"",item.getColor(),"","","","","","","","","","","","","","","","","","","","","","","","","","","","","",getOfferText(item),"","",item.getItemStatus().equals(status.PHASED_OUT) ? "Delete": "",hardDiskFormat,"credit_card|debit_card|net_banking|emi"+ (Double.valueOf(250D).compareTo(item.getSellingPrice()) <= 0&& Double.valueOf(25000D).compareTo(item.getSellingPrice()) >= 0 ? "|cash_on_delivery": "") }, "\t")).append("\n");}}}laptopFileWriter.write(tsvBuffer.toString());laptopFileWriter.flush();tsvBuffer.setLength(0);if (!isFirstPass || generationType.equals("ALL")) {laptopFileWriter.close();System.out.println("INFO : To generate Products List for Amazon type command : java AmazonSCDataGenerator products/prices");}return;}private void generateCameraData(List<Entity> entities, String generationType)throws Exception {StringBuffer tsvBuffer = new StringBuffer();if (isFirstPass) {tsvBuffer.append(StringUtils.join(new String[] {"Amazon.com Product Ads Header","Purge-Replace=false", "", "", "", "", "", "", "","", "", "", "", "", "", "", "", "", "", "", "", "","", "", "", "", "", "", "", "", "", "", "", "", "","", "", "", "", "", "", "", "", "", "", "", "", "","", "", "", "", "", "", "", "", "", "", "", "", "","", "", "", "", "", "", "", "", "", "", "", "", "","", "", "", "", "", "", "" }, "\t")).append("\n");tsvBuffer.append(StringUtils.join(new String[] { "SKU", "Title", "Link","Price", "Delivery Time","Recommended Browse Node", "Standard Product ID","Product ID Type", "Category", "Brand","Manufacturer", "Mfr part number", "Model Number","Description", "Shipping Cost", "Image","List Price", "Availability", "Optical Zoom","Digital Zoom", "Megapixels", "Display size","Memory Card Type", "Viewfinder type","Flash type", "Image Stabilization","Min Focal Length", "Max Focal Length","Mounting Type", "Offer note", "Discount Amount","Junglee Exclusive Offer", "Offer Promo Code","Accepted Payment Methods", "Screen Resolution","Display Technology", "Memory Storage Capacity","Colour Name", "Colour Map","Item package quantity", "Warranty","Assembly required", "Battery Cell Composition","Batteries Included", "Batteries Required","Power Source", "Power Adapter Included","Shipping Weight", "Weight", "Length", "Height","Width", "Update Delete" }, "\t")).append("\n");cameraFileWriter.write(tsvBuffer.toString());cameraFileWriter.flush();tsvBuffer.setLength(0);}for (Entity entity : entities) {if (Catalog.getInstance().getDefinitionsContainer().getCategory(entity.getCategoryID()) == null)continue;long categoryId = entity.getCategoryID();if (entity == null || entity.getSlides() == null|| (categoryId != 11002 && categoryId != 11003)) {continue;}Map<Long, String> fvalues = extractFeatureValuesForEntity(entity);List<Item> items = getItemsByCatalogId(entity.getID());Map<Long, String> itemIdImageUrlMap = getAllImageUrls(entity, items);CollectionUtils.filter(items, predicateActivePaused);if (CollectionUtils.isEmpty(items))continue;String warranty = StringUtils.defaultString(fvalues.get(AmazonFeatures.WARRANTY.getValue()));String batteryType = StringUtils.defaultString(fvalues.get(AmazonFeatures.BATTERY_TYPE.getValue()));String weight = StringUtils.defaultString(fvalues.get(AmazonFeatures.WEIGHT.getValue()));String size = StringUtils.defaultString(fvalues.get(AmazonFeatures.SIZE.getValue()));String length = "", width = "", height = "";if (!(size.equals("Not available") || size.equals(""))) {String[] list = size.split("\\s+");length = list[0] + " MM";width = list[2] + " MM";height = list[4] + " MM";}String flashmount = StringUtils.defaultString(fvalues.get(120392l)).split(" ")[0];if (flashmount.equals("Sony")) {flashmount = "Minoltabayonet";} else if (!flashmount.equals("")) {flashmount = flashmount.concat("bayonet");}for (Item item : items) {String productId = "";String productType = "";Long itemId = item.getId();populateItemMap(item);List<String> itemNumbers = getItemNumbers(itemId);if (!itemNumbers.isEmpty()) {for (String itemNumber : itemNumbers) {itemNumber = itemNumber.replaceAll("-", "");try {Long.parseLong(itemNumber);} catch (NumberFormatException nfe) {continue;}if (itemNumber.length() == 13) {productType = "EAN";} else if (itemNumber.length() == 12) {productType = "UPC";}productId = itemNumber;break;}}if (item.getColor() == null) {item.setColor("");}String sensorResolution = "";String viewFinder = "none";if (categoryId == 11002l) {sensorResolution = StringUtils.defaultString(fvalues.get(120026l));String vf = StringUtils.defaultString(fvalues.get(120342));if (vf == "Optical viewfinder") {viewFinder = "optical";} else if (vf == "Electronic viewfinder") {viewFinder = "digital";}} else {viewFinder = "optical";}String focalString = StringUtils.defaultString(fvalues.get(120336l));Matcher m = patt.matcher(focalString);int i = 0;String[] focalLength = { "", "" };while (i < 2 && m.find()) {focalLength[i] = m.group();i++;}String imageUrl = itemIdImageUrlMap.get(itemId);if (categoryId != 10073) {tsvBuffer.append(StringUtils.join(new String[] {itemId.toString(),getTitle(item),getUrl(item, "cameras"),"" + (int) (item.getSellingPrice()),"1",categoryId == 11002 ? "802863031": "802865031",productId,// productId,productType, // productType,"CamerasAndAccessories",item.getBrand(),item.getBrand(),"",item.getModelNumber(),StringUtils.defaultString(fvalues.get(AmazonFeatures.DESCRIPTION.getValue())),"0",imageUrl,""+ (item.getMrp() == 0 ? (int) item.getSellingPrice(): (int) (item.getMrp())),getAvailability(item),StringUtils.defaultString(fvalues.get(120332l)),// Optical// ZoomStringUtils.defaultString(fvalues.get(120333l)),// Digital// ZoomsensorResolution, // MegapixelsStringUtils.defaultString(fvalues.get(120005l)),// Display// Size"",// Memory card type// StringUtils.defaultString(fvalues.get(120355l)).replaceAll("--",// ", "),//Memory Card typesviewFinder,"other",fvalues.containsKey(120374l) ? "yes": "no", // image// stabilizationfocalLength[0],focalLength[1],flashmount,getOfferText(item),"credit_card|debit_card|net_banking|emi"+ (Double.valueOf(250D).compareTo(item.getSellingPrice()) <= 0&& Double.valueOf(25000D).compareTo(item.getSellingPrice()) >= 0 ? "|cash_on_delivery": ""),getResolution(Integer.parseInt(StringUtils.defaultString(fvalues.get(120340l).split(" ")[0],"0"))),StringUtils.defaultString(fvalues.get(120091l)).split(" ")[0],"",item.getColor(),"","",warranty.equals("") ? "": warranty+ " manufacturer's warranty","false",normalizeBatteryType(batteryType),"true", "true", "battery-powered","true", "", weight, length, height,width, "" }, "\t")).append("\n");}}}cameraFileWriter.write(tsvBuffer.toString());cameraFileWriter.flush();tsvBuffer.setLength(0);if (!isFirstPass || generationType.equals("ALL")) {cameraFileWriter.close();}return;}private static String getScreenResolution(String scrResolution) {if (StringUtils.isEmpty(scrResolution)) {return "";}try {return scrResolution.split(":")[1].trim();} catch (Exception e) {String[] splitString = scrResolution.split(" ");return splitString[0] + "X" + splitString[2];}}private static String getWirelessType(String wirelessType) {// TODO Auto-generated method stubwirelessType = StringUtils.remove(wirelessType, "/");wirelessType = StringUtils.join(StringUtils.splitByCharacterType(wirelessType), "_");wirelessType = StringUtils.remove(wirelessType, "._");return wirelessType.toUpperCase();}private static String getScreenSize(String screenSize) {return screenSize.split(" ")[0];}private static String getOS(String os) {if (StringUtils.contains(os, "Linux"))return "Linux";if (StringUtils.contains(os, "DOS"))return "DOS";if (StringUtils.contains(os, "Windows 7 Starter"))return "Windows 7 Starter";if (StringUtils.contains(os, "Windows 7 Basic"))return "Windows 7 Home Basic";if (StringUtils.contains(os, "Windows 7 Ultimate"))return "Windows 7 Ultimate";if (StringUtils.contains(os, "Windows 7 Premium"))return "Windows 7 Home Premium";if (StringUtils.contains(os, "Windows 7 Enterprise"))return "Windows 7 Enterprise";if (StringUtils.contains(os, "Windows 7 Professional"))return "Windows 7 Professional";if (StringUtils.contains(os, "Mac OS X"))return "Mac OS X";elsereturn "";}private static String getType(String processorType) {if (StringUtils.contains(processorType, "Core 2 Duo"))return "core_2_duo";if (StringUtils.contains(processorType, "AMD Athlon II"))return "athlon_64_x2";if (StringUtils.contains(processorType, "Intel Atom"))return "intel_atom";if (StringUtils.contains(processorType, "Pentium"))return "pentium";if (StringUtils.contains(processorType, "Celeron"))return "pentium";if (StringUtils.contains(processorType, "Celeron"))return "celeron";if (StringUtils.contains(processorType, "Celeron"))return "celeron";if (StringUtils.contains(processorType, "Intel Core"))return "intel_core_solo";if (StringUtils.contains(processorType, "AMD"))return "turion_64";elsereturn processorType;}private void generateMobileAccessories(List<Entity> entities,String generationType) throws Exception {StringBuffer tsvStringBuffer = new StringBuffer();// filter only phone related entitiesfor (Entity entity : entities) {List<Item> items = getItemsByCatalogId(entity.getID());Category category = Catalog.getInstance().getDefinitionsContainer().getCategory(entity.getCategoryID());if (category == null) {continue;}if (entity == null || entity.getSlides() == null|| category.getParentCategory().getID() != 10011) {continue;}AmazonCategories azCategory = null;Long categoryId = category.getID();if (categoryId.equals(10012L)) {azCategory = AmazonCategories.MOBILE_ACCESSORY_BLUETOOTH_HEADSETS;} else if (categoryId.equals(10013L)) {azCategory = AmazonCategories.ACCESSORY_MICROSD;} else if (categoryId.equals(10014L)) {azCategory = AmazonCategories.MOBILE_ACCESSORY_BATTERIES;} else if (categoryId.equals(10015L)) {azCategory = AmazonCategories.MOBILE_ACCESSORY_HEADSETS;} else if (categoryId.equals(10016L)) {azCategory = AmazonCategories.MOBILE_ACCESSORY_MAINS_CHARGERS;} else if (categoryId.equals(10017L)) {azCategory = AmazonCategories.COMPUTER_ACCESSORY_FLASH_DRIVE;} else if (categoryId.equals(10018L) || categoryId.equals(10021L)|| categoryId.equals(10022L)) {azCategory = AmazonCategories.MOBILE_ACCESSORY_COVERS;} else if (categoryId.equals(10019L)) {azCategory = AmazonCategories.MOBILE_ACCESSORY_CAR_CHARGERS;} else if (categoryId.equals(10020L)) {azCategory = AmazonCategories.MOBILE_ACCESSORY_SCREEN_GUARDS;} else if (categoryId.equals(10026L)) {azCategory = AmazonCategories.MOBILE_ACCESSORY_SPEAKERS;} else {continue;}Map<Long, String> fvalues = extractFeatureValuesForEntity(entity);Map<Long, String> itemIdImageUrlMap = getAllImageUrls(entity, items);String metaKeywords = StringUtils.defaultString(fvalues.get(AmazonFeatures.META_KEYWORDS.getValue()));List<String> keywords = new ArrayList<String>();String bulletsString = StringUtils.defaultString(fvalues.get(120081L));List<String> bullets = new ArrayList<String>();for (String keyword : bulletsString.split("--")) {bullets.add(keyword.trim().replaceAll("\\s+", " "));}for (String keyword : metaKeywords.split(",")) {keywords.add(keyword.trim());}int length1 = keywords.size();if (length1 < 5) {while (length1 < 5) {keywords.add("");length1 = length1 + 1;}}int length2 = bullets.size();if (length2 < 5) {while (length2 < 5) {bullets.add("");length2 = length2 + 1;}}CollectionUtils.filter(items, predicateActivePaused);if (CollectionUtils.isEmpty(items))continue;String storageCapacity = "";String warranty = StringUtils.defaultString(fvalues.get(120125L))+ " " + StringUtils.defaultString(fvalues.get(120219L));String batteryType = StringUtils.defaultString(fvalues.get(120103L));String batteryCap = StringUtils.defaultString(fvalues.get(120104L));if (azCategory.equals(AmazonCategories.COMPUTER_ACCESSORY_FLASH_DRIVE)) {storageCapacity = StringUtils.defaultString(fvalues.get(120112L));}if (azCategory.equals(AmazonCategories.MOBILE_ACCESSORY_BATTERIES)) {}if (azCategory.equals(AmazonCategories.ACCESSORY_MICROSD)) {storageCapacity = StringUtils.defaultString(fvalues.get(120099L));}for (Item item : items) {String productType = "";String productId = "";Long itemId = item.getId();List<String> itemNumbers = getItemNumbers(itemId);if (!itemNumbers.isEmpty()) {for (String itemNumber : itemNumbers) {itemNumber = itemNumber.replaceAll("-", "");try {Long.parseLong(itemNumber);} catch (NumberFormatException nfe) {continue;}if (itemNumber.length() == 13) {productType = "EAN";} else if (itemNumber.length() == 12) {productType = "UPC";}productId = itemNumber;break;}}populateItemMap(item);tsvStringBuffer.append(StringUtils.join(new String[] {new Long(item.getId()).toString(),getTitle(item),getUrl(item, "mobile-accessories"),"" + (int) (item.getSellingPrice()),"1",azCategory == null ? "" : azCategory.getValue().toString(),productId,productType,"Wireless",item.getBrand(),item.getBrand(),"",item.getModelNumber(),item.getFeatureDescription(),"0",itemIdImageUrlMap.get(item.getId()),""+ (item.getMrp() == 0 ? (int) item.getSellingPrice(): (int) (item.getMrp())),getAvailability(item),"","","","","","","","","","","","","","",storageCapacity,"","",item.getColor(),"","",warranty,"",batteryType,"","","","","","","","","",keywords.get(0),keywords.get(1),keywords.get(2),keywords.get(3),keywords.get(4),bullets.get(0),bullets.get(1),bullets.get(2),bullets.get(3),bullets.get(4),"","","","","",getOfferText(item),"","",item.getItemStatus().equals(status.PHASED_OUT) ? "Delete": "","credit_card|debit_card|net_banking|emi"+ (Double.valueOf(250D).compareTo(item.getSellingPrice()) <= 0&& Double.valueOf(25000D).compareTo(item.getSellingPrice()) >= 0 ? "|cash_on_delivery": "") }, "\t")).append("\n");}}productFileWriter.write(tsvStringBuffer.toString());productFileWriter.flush();tsvStringBuffer.setLength(0);}public void generateMobileData(List<Entity> entities, String generationType)throws Exception {System.out.println("INFO : Generating Products List for Amazon");StringBuffer tsvStringBuffer = new StringBuffer();if (isFirstPass) {tsvStringBuffer.append(generateTSVHeader());productFileWriter.write(tsvStringBuffer.toString());productFileWriter.flush();tsvStringBuffer.setLength(0);}// filter only phone related entitiesfor (Entity entity : entities) {if (Catalog.getInstance().getDefinitionsContainer().getCategory(entity.getCategoryID()) == null) {continue;}if (entity == null|| entity.getSlides() == null|| Catalog.getInstance().getDefinitionsContainer().getCategory(entity.getCategoryID()).getParentCategory().getID() != 10001) {continue;}// Treat phone related entities now.// get Feature values for all entities required for amazon.Map<Long, String> fvalues = extractFeatureValuesForEntity(entity);List<Item> items = getItemsByCatalogId(entity.getID());CollectionUtils.filter(items, predicateActivePaused);Map<Long, String> itemIdImageUrlMap = getAllImageUrls(entity, items);if (CollectionUtils.isEmpty(items))continue;String screenSize = StringUtils.defaultString(fvalues.get(AmazonFeatures.SCREEN_SIZE_IN.getValue()));String weight = StringUtils.defaultString(fvalues.get(AmazonFeatures.WEIGHT.getValue()));String screenResolution = StringUtils.defaultString(fvalues.get(AmazonFeatures.SCREEN_RESOLUTION.getValue()));String screenType = StringUtils.defaultString(fvalues.get(AmazonFeatures.SCREEN_TYPE.getValue()));String touchscreenType = StringUtils.defaultString(fvalues.get(AmazonFeatures.TOUCHSCREEN_TYPE.getValue()));String multipleSIM = StringUtils.defaultString(fvalues.get(AmazonFeatures.MULTIPLE_SIM.getValue()));String network3G = StringUtils.defaultString(fvalues.get(AmazonFeatures.NETWORK_3G.getValue()));String opsys = StringUtils.defaultString(fvalues.get(AmazonFeatures.OS.getValue()));String formFactor = StringUtils.defaultString(fvalues.get(AmazonFeatures.FORM_FACTOR.getValue()));String warranty = StringUtils.defaultString(fvalues.get(AmazonFeatures.WARRANTY.getValue()));String batteryType = StringUtils.defaultString(fvalues.get(AmazonFeatures.BATTERY_TYPE.getValue()));String size = StringUtils.defaultString(fvalues.get(AmazonFeatures.SIZE.getValue()));String metaKeywords = StringUtils.defaultString(fvalues.get(AmazonFeatures.META_KEYWORDS.getValue()));String color = StringUtils.defaultString(fvalues.get(AmazonFeatures.COLOR.getValue())).split("--")[0];if (!StringUtils.isEmpty(screenSize)) {screenSize = screenSize.split("\\s+")[0];}if (!StringUtils.isEmpty(screenResolution)) {screenResolution = screenResolution.substring(0,screenResolution.lastIndexOf(" "));}screenType = normalizeScreenType(screenType);String cellularTechnology = getCellularTechnology(multipleSIM,network3G);opsys = normalizeOperatingSystem(opsys);String userInput = "keypad";if (touchscreenType != "") {userInput = "touch_screen";}formFactor = normalizeFormFactor(formFactor);if (StringUtils.isNotEmpty(warranty)) {warranty = warranty + " manufacturer warranty";}batteryType = normalizeBatteryType(batteryType);String[] weightParts = weight.split("\\s+");if (weightParts.length > 1) {weight = weightParts[0];try {Float.parseFloat(weight);} catch (NumberFormatException e) {System.out.println(entity.getID()+ "has invalid weight format");// e.printStackTrace();weight = "";}}String length = "", width = "", height = "";if (!(size.equals("Not available") || size.equals(""))) {String[] list = size.split("\\s+");length = list[0] + " MM";width = list[2] + " MM";height = list[4] + " MM";}List<String> keywords = new ArrayList<String>();for (String keyword : metaKeywords.split(",")) {keywords.add(keyword.trim());}int length1 = keywords.size();if (length1 < 5) {while (length1 < 5) {keywords.add("");length1 = length1 + 1;}}for (String keyword : keywords) {if (keyword.length() > 50)System.out.println(keyword);}for (Item item : items) {Long itemId = item.getId();populateItemMap(item);List<String> itemNumbers = getItemNumbers(itemId);String productId = "";String productType = "";if (!itemNumbers.isEmpty()) {for (String itemNumber : itemNumbers) {itemNumber = itemNumber.replaceAll("-", "");try {Long.parseLong(itemNumber);} catch (NumberFormatException nfe) {continue;}if (itemNumber.length() == 13) {productType = "EAN";} else if (itemNumber.length() == 12) {productType = "UPC";}productId = itemNumber;if (!productType.equals("")) {break;}}}// duplicate page creationif (itemId > 21000) {productId = dupItemNumbersMap.get(itemId);if (productId.length() == 13) {productType = "EAN";} else if (productId.length() == 12) {productType = "UPC";} else {productType = "ASIN";}}if (item.getColor() != null) {color = item.getColor();}String imageUrl = itemIdImageUrlMap.get(itemId);tsvStringBuffer.append(StringUtils.join(new String[] {new Long(item.getId()).toString(),getTitle(item),getUrl(item, "mobile-phones"),"" + (int) (item.getSellingPrice()),"1","803073031",productId,productType,"Wireless",item.getBrand(),item.getBrand(),"",item.getModelNumber(),StringUtils.defaultString(fvalues.get(AmazonFeatures.DESCRIPTION.getValue())),"0",imageUrl,""+ (item.getMrp() == 0 ? (int) item.getSellingPrice(): (int) (item.getMrp())),getAvailability(item),opsys,"","",StringUtils.defaultString(fvalues.get(120026L)).split(" ")[0],// Digital zoom,// MegapixelscreenSize,cellularTechnology,getMinutesForTalktime(fvalues),// TalktimegetHoursForStandBy(fvalues),// Standby// time"", // charging timeuserInput,"", // device typeformFactor,"", // connectivity Technologies"", // Special featuresStringUtils.defaultString(fvalues.get(AmazonFeatures.BUILTIN_MEMORY.getValue())),"",// Optical zoom"", // Memory Card Typecolor,"", // color map"1",warranty,"false",// assemblybatteryType,"true", // Batteries Included"true", // Batteries Required"battery-powered", // Power Source"true", // Power Adapter inculded"", // shipping weightweight, // Weightlength,height,width,// StringUtils.defaultString(fvalues.get(AmazonFeatures.RAM_MEMORY.getValue())),// "", "",keywords.get(0),keywords.get(1),keywords.get(2),keywords.get(3),keywords.get(4),"","","","","","","","","","",getOfferText(item),"","",item.getItemStatus().equals(status.PHASED_OUT) ? "Delete": "","credit_card|debit_card|net_banking|emi"+ (Double.valueOf(250D).compareTo(item.getSellingPrice()) <= 0&& Double.valueOf(25000D).compareTo(item.getSellingPrice()) >= 0 ? "|cash_on_delivery": "") }, "\t")).append("\n");}}productFileWriter.write(tsvStringBuffer.toString());productFileWriter.flush();tsvStringBuffer.setLength(0);generateMobileAccessories(entities, generationType);if (!isFirstPass || generationType.equals("ALL")) {productFileWriter.close();System.out.println("INFO : Products.tsv successfully created at "+ Utils.EXPORT_PATH + " successfully generated.");GmailUtils gm = new GmailUtils();// "rajveer.singh@shop2020.in"String[] sendTo = { "amit.gupta@shop2020.in" };try {gm.sendSSLMessage(sendTo,"Amazon Products Content Generation Successful ? : true","Content generation completed at time : "+ Calendar.getInstance().getTime().toString(),"build@shop2020.in", "cafe@nes", new ArrayList<File>());} catch (MessagingException e) {System.out.println("Could not send status mail" + e);}}}private List<Item> getItemsByCatalogId(long id) throws Exception {try {return cc.getItemsByCatalogId(id);} catch (Exception e) {cc = new CatalogClient().getClient();return cc.getItemsByCatalogId(id);}}private List<String> getItemNumbers(Long itemId) throws Exception {try {return whc.getItemNumbers(itemId);} catch (Exception e) {whc = new WarehouseClient().getClient();return whc.getItemNumbers(itemId);}}public void generateSCProdData() throws Exception {File file = new File(Utils.CONTENT_DB_PATH + objFile);if (file.exists()) {try {// use bufferingInputStream fileInputStream = new FileInputStream(file);InputStream buffer = new BufferedInputStream(fileInputStream);ObjectInput input = new ObjectInputStream(buffer);try {// deserialize the ListitemMapOld = (Map<Long, Map<String, Object>>) input.readObject();// display its data} finally {input.close();}} catch (Exception ex) {System.out.println("Error reading file");}} else {itemMapOld = new HashMap<Long, Map<String, Object>>();}if (!generationType.equals("ALL")) {itemMap = itemMapOld;} else {itemMap = new HashMap<Long, Map<String, Object>>();}generateMobileData(this.validEntites, generationType);generateLaptopData(this.validEntites, generationType);generateCameraData(this.validEntites, generationType);Set<Entity> entities1 = new HashSet<Entity>();if (!generationType.equals("ALL")) {isFirstPass = false;// get All entities from ItemIdsCatalogClient cclient = new CatalogClient();Client catalogclient = cclient.getClient();Iterator<Map.Entry<Long, Map<String, Object>>> iter = itemMap.entrySet().iterator();while (iter.hasNext()) {Map.Entry<Long, Map<String, Object>> entry = iter.next();Item item = catalogclient.getItem(entry.getKey());// This item in the map doesnt exists in the system anymore// remove it from the mapif (item == null) {System.out.println("Couldnt find --" + entry.getKey()+ "-- in catalog");iter.remove();continue;}if (item.getItemStatus() == null) {System.out.println("Couldnt find --" + entry.getKey()+ "-- in catalog");System.out.println(item);}if (item.getItemStatus().equals(status.PHASED_OUT)) {iter.remove();continue;}Map<String, Object> valMap = entry.getValue();String offerText = getOfferText(item);if (!offerText.equals(" ")) {if (!(((Double) valMap.get("price")).compareTo(item.getSellingPrice()) == 0&& getAvailability(item).equals((String) valMap.get("availability")) && StringUtils.equals((String) valMap.get("offer"), offerText))) {valMap.put("price", item.getSellingPrice());valMap.put("availability", getAvailability(item));valMap.put("offer", offerText);entities1.add(CreationUtils.getEntity(item.getCatalogItemId()));}}}List<Entity> entityList = new ArrayList<Entity>(entities1);predicateActivePaused = new Predicate() {@Overridepublic boolean evaluate(Object obj) {Item item = (Item) obj;return Arrays.asList(new status[] { status.ACTIVE, status.PAUSED,status.PAUSED_BY_RISK, status.PHASED_OUT }).contains(item.getItemStatus());}};generateMobileData(entityList, generationType);generateLaptopData(entityList, generationType);generateCameraData(entityList, generationType);}// Put itemMap to fileObjectOutputStream os = new ObjectOutputStream(new FileOutputStream(Utils.CONTENT_DB_PATH + objFile));os.writeObject(itemMap);os.close();}private void populateItemMap(Item item) throws PromotionException,TException {if (isFirstPass) {Map<String, Object> valueMap;Long itemId = item.getId();if (!itemMap.containsKey(itemId)) {valueMap = new HashMap<String, Object>();// ignore the duplicate items from the listif (!dupItemsMap.containsValue(itemId)) {itemMap.put(itemId, valueMap);}} else {valueMap = itemMap.get(itemId);}String availability = getAvailability(item);valueMap.put("price", item.getSellingPrice());valueMap.put("availability", availability);valueMap.put("offer", getOfferText(item));}}private String getAvailability(Item item) {String availability = "0";if (item.getItemStatus().equals(status.ACTIVE)) {if (!(item.isRisky() && PAUSED_BY_RISK_ITEMS.contains(item.getId()))) {availability = "1";}}return availability;}private static StringBuffer generateTSVHeader() {StringBuffer tsvStringBuffer = new StringBuffer();tsvStringBuffer.append(StringUtils.join(new String[] {"Amazon.com Product Ads Header", "Purge-Replace=false","", "", "", "", "", "", "", "", "", "", "", "", "", "","", "", "", "", "", "", "", "", "", "", "", "", "", "","", "", "", "", "", "", "", "", "", "", "", "", "", "","", "", "", "", "", "", "", "", "", "", "", "", "", "","", "", "", "", "", "", "", "", "", "", "" }, "\t")).append("\n");tsvStringBuffer.append(StringUtils.join(new String[] { "SKU", "Title", "Link","Price", "Delivery Time", "Recommended Browse Node","Standard Product ID", "Product ID Type", "Category","Brand", "Manufacturer", "Mfr part number","Model Number", "Description", "Shipping Cost","Image", "List Price", "Availability","Operating System", "Wireless Type", "Digital zoom","Megapixels", "Display size", "Cellular Technology","Talk Time", "Standby Time", "Charging Time","User Input", "Device Type", "Form Factor","Connectivity Technology", "Special Features","Memory Storage Capacity", "Optical Zoom","Memory Card Type", "Colour Name", "Colour Map","Item Package Quantity", "Warranty","Assembly required", "Battery Cell Composition","Batteries Included", "Batteries Required","Power Source", "Power Adapter Included","Shipping Weight", "Weight", "Length", "Height","Width", "Keywords1", "Keywords2", "Keywords3","Keywords4", "Keywords5", "Bullet point1","Bullet point2", "Bullet point3", "Bullet point4","Bullet point5", "Other image-url1","Other image-url2", "Other image-url3","Other image-url4", "Other image-url5", "Offer note","Discount Amount", "Junglee Exclusive Offer","Offer Promo Code", "Is Gift Wrap Available","Registered Parameter", "Update Delete","Accepted Payment Methods" }, "\t")).append("\n");return tsvStringBuffer;}public static void generatePricesAndAvailability(List<Item> items)throws Exception {System.out.println("INFO : Generating Products Prices and Availability for Amazon");StringBuffer tsvStringBuffer = new StringBuffer();tsvStringBuffer.append("Amazon.com Product Ads Header\tPurge-Replace=false\nsku\tprice\tAvailability\tdelivery_time\tUpdate Delete\n");for (Item item : items) {Long itemId = item.getId();String price = "" + (int) (item.getSellingPrice());if (item.isRisky() && PAUSED_BY_RISK_ITEMS.contains(item.getId())) {continue;}tsvStringBuffer.append(itemId + "\t" + price + "\tTRUE\t1\tpartialUpdate\n");}String fileName = "PricesAndAvailabilityONE.tsv";String filePath = Utils.EXPORT_PATH + fileName;File f1 = new File(filePath);FileWriter writer1 = new FileWriter(f1);writer1.write(tsvStringBuffer.toString());writer1.close();System.out.println("INFO : PricesAndAvailability.tsv successfully created at /home/amit/Desktop/cms/ successfully generated.");System.out.println("INFO : To generate Products List for Amazon type command : java AmazonSCDataGenerator products/prices");ConfigClient configClient = ConfigClient.getClient();String isFtpUpload = configClient.get("UPLOAD_AMAZON_CONTENT_FTP");// upload the file to Amazon ftp location only when app is run from// stagingSystem.out.println("Is ftp Upload:" + isFtpUpload);if (isFtpUpload.equals("TRUE")) {try {upload(FTP_UPLOAD_URL,FTP_USER_NAME, FTP_PASSWORD, filePath, fileName);} catch(Exception e){e.printStackTrace();System.out.println("Couldnt upload" + e);// Eventually other operations here ...GmailUtils gm = new GmailUtils();// "rajveer.singh@shop2020.in",String[] sendTo = { "amit.gupta@shop2020.in" };try {gm.sendSSLMessage(sendTo, "FTP upload failed","Upload time : "+ Calendar.getInstance().getTime().toString(),"build@shop2020.in", "cafe@nes", Utils.EXPORT_PATH+ "PricesAndAvailabilityONE.tsv");} catch (MessagingException me) {System.out.println("Could not send status mail" + me);}}}return;}public static void generatePricesAndAvailability(boolean cmd) throws Exception {List<Long> allAccessories = Arrays.asList(10002l,10003l,10004l,10005l,10006l,10010l,10050l,11002l,11003l,12002l,12101l);CatalogClient cc = new CatalogClient();Client cl = cc.getClient();System.out.println("INFO : Generating Products Prices and Availability for Amazon");StringBuffer tsvStringBuffer = new StringBuffer();tsvStringBuffer.append("Amazon.com Product Ads Header\tPurge-Replace=false\nsku\tprice\tAvailability\tdelivery_time\tUpdate Delete\tDiscount Amount\tSale Start Date\tSale End Date\tOffer Promo Code\tOffer note\n");List<Item> items = cl.getAllItemsByStatus(status.ACTIVE);for (Item item : items) {if(allAccessories.contains(item.getCategory())){continue;}Long itemId = item.getId();String price = "" + (int) (item.getSellingPrice());String discount = "" + (int)Math.min(Math.round(item.getSellingPrice()/10d), 500d);if (item.isRisky() && PAUSED_BY_RISK_ITEMS.contains(item.getId())) {tsvStringBuffer.append(itemId + "\t" + price + "\tFALSE\t1\tpartialUpdate\t" + discount + "\t10/17/2013\t11/12/2013\tsaholicdiwali\tNA\n");} else {tsvStringBuffer.append(itemId + "\t" + price + "\tTRUE\t1\tpartialUpdate\t" + discount + "\t10/17/2013\t11/12/2013\tsaholicdiwali\tNA\n");}}items.clear();items.addAll(cl.getAllItemsByStatus(status.PAUSED));items.addAll(cl.getAllItemsByStatus(status.PHASED_OUT));for (Item item: items){if(allAccessories.contains(item.getCategory())){continue;}Long itemId = item.getId();String price = "" + (int) (item.getSellingPrice());String discount = "" + (int)Math.min(Math.round(item.getSellingPrice()/10d), 500d);tsvStringBuffer.append(itemId + "\t" + price + "\tFALSE\t1\tpartialUpdate\t" + discount + "\t10/17/2013\t11/12/2013\tsaholicdiwali\tNA\n");}String fileName = "PricesAndAvailability.tsv";String filePath = Utils.EXPORT_PATH + fileName;File f1 = new File(filePath);FileWriter writer1 = new FileWriter(f1);writer1.write(tsvStringBuffer.toString());writer1.close();System.out.println("INFO : PricesAndAvailability.tsv successfully created at /home/amit/Desktop/cms/ successfully generated.");System.out.println("INFO : To generate Products List for Amazon type command : java AmazonSCDataGenerator products/prices");ConfigClient configClient = ConfigClient.getClient();String isFtpUpload = configClient.get("UPLOAD_AMAZON_CONTENT_FTP");// upload the file to Amazon ftp location only when app is run from// stagingSystem.out.println("Is ftp Upload:" + isFtpUpload);if (isFtpUpload.equals("TRUE") && cmd) {System.out.println("Before upload call");try{upload(FTP_UPLOAD_URL,FTP_USER_NAME, FTP_PASSWORD, filePath, fileName);} catch (Exception e1){System.out.println(e1.getMessage());e1.printStackTrace();// Eventually other operations here ...GmailUtils gm = new GmailUtils();// "rajveer.singh@shop2020.in",String[] sendTo = { "amit.gupta@shop2020.in" };try {gm.sendSSLMessage(sendTo, "Failed to upload amazon Prices and Availability","Reupload prices and availability","build@shop2020.in", "cafe@nes", Utils.EXPORT_PATH+ "PricesAndAvailability.tsv");} catch (MessagingException e) {System.out.println("Could not send status mail" + e);}}}return;}private String getOfferText(Item item) {// try loading promotion clientString couponString = "\t\t\t";String offerString = "";Long itemId = item.getId();List<Long> l = new ArrayList<Long>();l.add(itemId);List<ItemCouponDiscount> icd = null;try {try {icd = pc.getItemDiscountMap(l);} catch (Exception e) {Logger.log("Error:", e);pc = new PromotionClient().getClient();icd = pc.getItemDiscountMap(l);}} catch (Exception e) {log.error("Got error for item " + itemId);log.error(e.getMessage());}if (icd != null && icd.size() > 0) {Double maxOffer = 0D;String couponCode = "";for (ItemCouponDiscount disc : icd) {if (maxOffer.compareTo(disc.getDiscount()) < 0) {maxOffer = disc.getDiscount();couponCode = disc.getCouponCode();}}couponString = MessageFormat.format(Utils.OFFER_MSG_TEMPLATE,couponCode, "" + (int) (maxOffer.doubleValue()));}if (!StringUtils.isEmpty(item.getBestDealText())) {offerString = item.getBestDealText();}return offerString + couponString;}private static String getHoursForStandBy(Map<Long, String> fvalues) {String standby = fvalues.get(AmazonFeatures.STANDBY_2G.getValue());if (standby == null) {standby = fvalues.get(AmazonFeatures.STANDBY_2G.getValue());}if (standby == null) {return "";}String[] dayshours = standby.split("\\D+");String days = "0";String hours = "0";if (dayshours.length > 0) {days = dayshours[0];if (dayshours.length > 1) {hours = dayshours[1];}}try {String s = String.valueOf(Integer.parseInt(days) * 24+ Integer.parseInt(hours));return s.equals("0") ? "" : s;} catch (Exception e) {return "";}}private static String getMinutesForTalktime(Map<Long, String> fvalues) {String talktime = fvalues.get(AmazonFeatures.TALKTIME_2G.getValue());if (talktime == null) {talktime = fvalues.get(AmazonFeatures.TALKTIME_3G.getValue());}if (talktime == null) {return "";}String[] hourMins = talktime.split("\\D+");String hours = "0";String mins = "0";if (hourMins.length > 0) {if (!hourMins[0].equals("")) {hours = hourMins[0];}if (hourMins.length > 3) {if (!hourMins[3].equals("")) {mins = hourMins[3];}}}try {String s = String.valueOf(Integer.parseInt(hours) * 60+ Integer.parseInt(mins));return s.equals("0") ? "" : s;} catch (Exception e) {return "";}}private String getResolution(int dots) {switch (dots) {case 307200:return "640 x 480";case 480000:return "800 x 600";case 614400:return "1024 x 600";case 786432:return "1024 x 768";case 995328:return "1152 x 864";case 921600:return "1280 x 720";case 983040:return "1280 x 768";case 1024000:return "1280 x 800";case 1228800:return "1280 x 960";case 1310720:return "1280 x 1024";case 1044480:return "1360 x 768";case 1049088:return "1366 x 768";case 1470000:return "1400 x 1050";case 1296000:return "1440 x 900";case 1440000:return "1600 x 900";case 1920000:return "1600 x 1200";case 1764000:return "1680 x 1050";case 2073600:return "1920 x 1080";case 2304000:return "1920 x 1200";case 2359296:return "2048 x 1152";case 3686400:return "2560 x 1440";case 4096000:return "2560 x 1600";case 671102:return "1093 x 614";case 966207:return "1311 x 737";default:return "";}}private static List<Long> getPausedByRiskItems() {try {Reader reader = new FileReader(Utils.EXPORT_PATH+ Utils.RISKY_PAUSED_JSON);return new Gson().fromJson(reader, new TypeToken<List<Long>>() {}.getType());} catch (FileNotFoundException e) {log.error("Could not read popularity file");e.printStackTrace();return new ArrayList<Long>();}}public static void upload(String hostName, String username,String password, String localFilePath, String remoteFilePath) {File file = new File(localFilePath);if (!file.exists())throw new RuntimeException("Error. Local file not found");StandardFileSystemManager manager = new StandardFileSystemManager();try {manager.init();// Create local file objectFileObject localFile = manager.resolveFile(file.getAbsolutePath());// Create remote file objectFileObject remoteFile = manager.resolveFile(createConnectionString(hostName, username, password,remoteFilePath), createDefaultOptions());// Copy local file to sftp serverremoteFile.copyFrom(localFile, Selectors.SELECT_SELF);System.out.println("File upload success");} catch (Exception e) {throw new RuntimeException(e);} finally {manager.close();}}public static String createConnectionString(String hostName,String username, String password, String remoteFilePath) {return "sftp://" + username + ":" + password + "@" + hostName + "/"+ remoteFilePath;}// Method to setup default SFTP config:public static FileSystemOptions createDefaultOptions()throws FileSystemException {// Create SFTP optionsFileSystemOptions opts = new FileSystemOptions();// SSH Key checkingSftpFileSystemConfigBuilder.getInstance().setStrictHostKeyChecking(opts, "no");// Root directory set to user homeSftpFileSystemConfigBuilder.getInstance().setUserDirIsRoot(opts, true);// Timeout is count by MillisecondsSftpFileSystemConfigBuilder.getInstance().setTimeout(opts, 10000);return opts;}}