Rev 17292 | Rev 20172 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | RSS feed
from BeautifulSoup import BeautifulSoupfrom datetime import datetimefrom dtr.utils.utils import to_java_date, fetchResponseUsingProxyfrom email.mime.multipart import MIMEMultipartfrom email.mime.text import MIMETextfrom scripts.AmazonBestSellers import getSoupObjectimport jsonimport optparseimport pymongoimport reimport smtplibimport urllib2from dtr.utils.PaytmOfferScraper import fetchOfferscon = Noneparser = optparse.OptionParser()parser.add_option("-m", "--m", dest="mongoHost",default="localhost",type="string", help="The HOST where the mongo server is running",metavar="mongo_host")(options, args) = parser.parse_args()exceptionList = []bestSellers = []now = datetime.now()class __RankInfo:def __init__(self, identifier, rank, category, available_price, gross_price, in_stock, coupon, thumbnail, source_product_name, marketPlaceUrl, cod):self.identifier = identifierself.rank = rankself.category = categoryself.available_price = available_priceself.gross_price = gross_priceself.in_stock = in_stockself.coupon = couponself.thumbnail = thumbnailself.source_product_name = source_product_nameself.marketPlaceUrl = marketPlaceUrlself.cod = coddef get_mongo_connection(host=options.mongoHost, port=27017):global conif con is None:print "Establishing connection %s host and port %d" %(host,port)try:con = pymongo.MongoClient(host, port)except Exception, e:print ereturn Nonereturn condef scrapeBestSellerMobiles():global bestSellersrank = 0mobileCategoryUrl = 'https://catalog.paytm.com/v1/g/electronics/mobile-accessories/mobiles?page_count=%d&items_per_page=25&sort_popular=1&cat_tree=1'for i in range(1,5):data = fetchResponseUsingProxy(mobileCategoryUrl%(i))jsonResponse = json.loads(data)for jsonProduct in jsonResponse['grid_layout']:rank = rank + 1identifier = str(jsonProduct['complex_product_id'])r_info = __RankInfo(identifier,rank, None, None,None,None,None,None,None,None,None)print rank, identifierbestSellers.append(r_info)def commitBestSellers(category):global exceptionListprint "Rank",print '\t',print 'Identifier'for x in bestSellers:print x.rank,print '\t',print x.identifier,print '\t',col = list(get_mongo_connection().Catalog.MasterData.find({'identifier':x.identifier, 'source_id':6}))print "count sku",len(col)print '\n'if len(col) == 0:x.category = categoryexceptionList.append(x)else:get_mongo_connection().Catalog.MasterData.update({'identifier':x.identifier, 'source_id':6 }, {'$set' : {'rank':x.rank,'updatedOn':to_java_date(now)}})def scrapeBestSellerTablets():global bestSellersbestSellers = []rank = 0bestSellerTabletsUrl = 'https://catalog.paytm.com/v1/g/electronics/mobile-accessories/headsets?page_count=%d&items_per_page=25&sort_popular=1'for i in range(1,5):data = fetchResponseUsingProxy(bestSellerTabletsUrl%(i))jsonResponse = json.loads(data)for jsonProduct in jsonResponse['grid_layout']:rank = rank + 1identifier = jsonProduct["url"].split("?")[0].split("/")[-1]r_info = __RankInfo(identifier,rank, None, None,None,None,None,None,None,None,None)print rank, identifierbestSellers.append(r_info)def resetRanks(category_id):get_mongo_connection().Catalog.MasterData.update({'rank':{'$gt':0},'source_id':6,'category_id':category_id}, {'$set':{'rank':0}}, upsert=False, multi=True)def sendEmail():message="""<html><body><h3>PayTM Best Sellers not in master</h3><table border="1" style="width:100%;"><thead><tr><th>Identifier</th><th>Category</th><th>Rank</th><th>Available_price</th><th>Gross_price</th><th>In_stock</th><th>Coupon</th><th>Thumbnail</th><th>Source_product_name</th><th>MarketPlaceUrl</th><th>Cod</th></tr></thead><tbody>"""for item in exceptionList:try:message+="""<tr><td style="text-align:center">"""+(item.identifier)+"""</td><td style="text-align:center">"""+(item.category)+"""</td><td style="text-align:center">"""+str(item.rank)+"""</td><td style="text-align:center">"""+str(item.available_price)+"""</td><td style="text-align:center">"""+str(item.gross_price)+"""</td><td style="text-align:center">"""+str(item.in_stock)+"""</td><td style="text-align:center">"""+str(item.coupon)+"""</td><td style="text-align:center">"""+str(item.thumbnail)+"""</td><td style="text-align:center">"""+str(item.source_product_name)+"""</td><td style="text-align:center">"""+str(item.marketPlaceUrl)+"""</td><td style="text-align:center">"""+str(item.cod)+"""</td></tr>"""except:continuemessage+="""</tbody></table></body></html>"""print message#recipients = ['amit.gupta@saholic.com']recipients = ['rajneesh.arora@saholic.com','kshitij.sood@saholic.com','chaitnaya.vats@saholic.com','manoj.kumar@saholic.com','amit.gupta@saholic.com']msg = MIMEMultipart()msg['Subject'] = "Paytm Best Sellers" + ' - ' + str(datetime.now())msg['From'] = ""msg['To'] = ",".join(recipients)msg.preamble = "Paytm Best Sellers" + ' - ' + str(datetime.now())html_msg = MIMEText(message, 'html')msg.attach(html_msg)smtpServer = smtplib.SMTP('localhost')smtpServer.set_debuglevel(1)sender = 'dtr@shop2020.in'try:smtpServer.sendmail(sender, recipients, msg.as_string())print "Successfully sent email"except:print "Error: unable to send email."def addExtraAttributes():for e in exceptionList:url = "https://catalog.paytm.com/v1/mobile/product/%s" %(e.identifier.strip())try:response_data = fetchResponseUsingProxy(url, proxy=False)except:continueinput_json = json.loads(response_data)offerPrice = float(input_json['offer_price'])e.cod = int(input_json['pay_type_supported'].get('COD'))offerUrl = (input_json['offer_url'])e.in_stock = (input_json['instock'])try:offers = fetchOffers(offerUrl)except:continuebestOffer = {}e.gross_price = float(offerPrice)effective_price = offerPricecoupon = ""for offer_data in offers.get('codes'):if effective_price > offer_data.get('effective_price'):effective_price = offer_data.get('effective_price')bestOffer = offer_datacoupon = bestOffer.get('code')e.source_product_name = input_json['name']e.thumbnail = input_json['thumbnail']e.marketPlaceUrl = input_json['shareurl']e.coupon = coupone.available_price = float(effective_price)def main():scrapeBestSellerMobiles()if len(bestSellers) > 0:resetRanks(3)commitBestSellers("MOBILE")scrapeBestSellerTablets()if len(bestSellers) > 0:resetRanks(5)commitBestSellers("TABLET")addExtraAttributes()sendEmail()if __name__=='__main__':main()