Rev 21135 | Blame | Compare with Previous | Last modification | View Log | RSS feed
from BeautifulSoup import BeautifulSoupfrom datetime import datetimefrom dtr.utils.utils import to_java_date, fetchResponseUsingProxyfrom email.mime.multipart import MIMEMultipartfrom email.mime.text import MIMETextimport jsonimport optparseimport pymongoimport reimport smtplibimport urllib2from dtr.utils.PaytmOfferScraper import fetchOffersfrom shop2020.utils.EmailAttachmentSender import get_attachment_partfrom shop2020.utils import EmailAttachmentSendercon = Noneparser = optparse.OptionParser()parser.add_option("-m", "--m", dest="mongoHost",default="localhost",type="string", help="The HOST where the mongo server is running",metavar="mongo_host")(options, args) = parser.parse_args()exceptionList = []bestSellers = []now = datetime.now()class __RankInfo:def __init__(self, identifier, rank, category, available_price, gross_price, in_stock, coupon, thumbnail, source_product_name, marketPlaceUrl, cod):self.identifier = identifierself.rank = rankself.category = categoryself.available_price = available_priceself.gross_price = gross_priceself.in_stock = in_stockself.coupon = couponself.thumbnail = thumbnailself.source_product_name = source_product_nameself.marketPlaceUrl = marketPlaceUrlself.cod = coddef get_mongo_connection(host=options.mongoHost, port=27017):global conif con is None:print "Establishing connection %s host and port %d" %(host,port)try:con = pymongo.MongoClient(host, port)except Exception, e:print ereturn Nonereturn condef scrapeBestSellerMobiles():global bestSellersrank = 0mobileCategoryUrl = 'https://catalog.paytm.com/v1/g/electronics/mobile-accessories/mobiles?page_count=%d&items_per_page=25&sort_popular=1&cat_tree=1'for i in range(1,5):data = fetchResponseUsingProxy(mobileCategoryUrl%(i))jsonResponse = json.loads(data)for jsonProduct in jsonResponse['grid_layout']:rank = rank + 1identifier = str(jsonProduct['complex_product_id'])r_info = __RankInfo(identifier,rank, None, None,None,None,None,None,None,None,None)print rank, identifierbestSellers.append(r_info)def commitBestSellers(category):global exceptionListprint "Rank",print '\t',print 'Identifier'for x in bestSellers:print x.rank,print '\t',print x.identifier,print '\t',col = list(get_mongo_connection().Catalog.MasterData.find({'identifier':x.identifier, 'source_id':6}))print "count sku",len(col)print '\n'if len(col) == 0:x.category = categoryexceptionList.append(x)else:get_mongo_connection().Catalog.MasterData.update({'identifier':x.identifier, 'source_id':6 }, {'$set' : {'rank':x.rank,'updatedOn':to_java_date(now)}})def scrapeBestSellerTablets():global bestSellersbestSellers = []rank = 0bestSellerTabletsUrl = 'https://catalog.paytm.com/v1/g/electronics/mobile-accessories/headsets?page_count=%d&items_per_page=25&sort_popular=1'for i in range(1,5):data = fetchResponseUsingProxy(bestSellerTabletsUrl%(i))jsonResponse = json.loads(data)for jsonProduct in jsonResponse['grid_layout']:rank = rank + 1identifier = str(jsonProduct['complex_product_id'])r_info = __RankInfo(identifier,rank, None, None,None,None,None,None,None,None,None)print rank, identifierbestSellers.append(r_info)def resetRanks(category_id):get_mongo_connection().Catalog.MasterData.update({'rank':{'$gt':0},'source_id':6,'category_id':category_id}, {'$set':{'rank':0}}, upsert=False, multi=True)def sendEmail():message="""<html><body><h3>PayTM Best Sellers not in master</h3><table border="1" style="width:100%;"><thead><tr><th>Identifier</th><th>Category</th><th>Rank</th><th>Available_price</th><th>Gross_price</th><th>In_stock</th><th>Coupon</th><th>Thumbnail</th><th>Source_product_name</th><th>MarketPlaceUrl</th><th>Cod</th></tr></thead><tbody>"""for item in exceptionList:try:message+="""<tr><td style="text-align:center">"""+(item.identifier)+"""</td><td style="text-align:center">"""+(item.category)+"""</td><td style="text-align:center">"""+str(item.rank)+"""</td><td style="text-align:center">"""+str(item.available_price)+"""</td><td style="text-align:center">"""+str(item.gross_price)+"""</td><td style="text-align:center">"""+str(item.in_stock)+"""</td><td style="text-align:center">"""+str(item.coupon)+"""</td><td style="text-align:center">"""+str(item.thumbnail)+"""</td><td style="text-align:center">"""+str(item.source_product_name)+"""</td><td style="text-align:center">"""+str(item.marketPlaceUrl)+"""</td><td style="text-align:center">"""+str(item.cod)+"""</td></tr>"""except:continuemessage+="""</tbody></table></body></html>"""print message#recipients = ['amit.gupta@saholic.com']recipients = ['kshitij.sood@saholic.com','ritesh.chauhan@saholic.com','aishwarya.singh@saholic.com']EmailAttachmentSender.mail_send_grid("dtr@smartdukaan.com","apikey", "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", recipients, "Paytm Best Sellers",message ,[],[],[])def addExtraAttributes():for e in exceptionList:url = "https://catalog.paytm.com/v1/mobile/product/%s" %(e.identifier.strip())try:response_data = fetchResponseUsingProxy(url, proxy=False)except:continueinput_json = json.loads(response_data)offerPrice = float(input_json['offer_price'])e.cod = int(input_json['pay_type_supported'].get('COD'))offerUrl = (input_json['offer_url'])e.in_stock = (input_json['instock'])try:offers = fetchOffers(offerUrl)except:continuebestOffer = {}e.gross_price = float(offerPrice)effective_price = offerPricecoupon = ""for offer_data in offers.get('codes'):if effective_price > offer_data.get('effective_price'):effective_price = offer_data.get('effective_price')bestOffer = offer_datacoupon = bestOffer.get('code')e.source_product_name = input_json['name']e.thumbnail = input_json['thumbnail']e.marketPlaceUrl = input_json['shareurl']e.coupon = coupone.available_price = float(effective_price)def main():scrapeBestSellerMobiles()if len(bestSellers) > 0:resetRanks(3)commitBestSellers("MOBILE")scrapeBestSellerTablets()if len(bestSellers) > 0:resetRanks(5)commitBestSellers("TABLET")addExtraAttributes()sendEmail()if __name__=='__main__':main()