Rev 20751 | Blame | Compare with Previous | Last modification | View Log | RSS feed
'''Created on Jan 15, 2015@author: amit'''from BeautifulSoup import BeautifulSoupfrom bson.binary import Binaryfrom datetime import datetime, date, timedeltafrom dtr import mainfrom dtr.config import PythonPropertyReaderfrom dtr.dao import AffiliateInfo, Order, SubOrderfrom dtr.main import getBrowserObject, ScrapeException, getStore, ParseException, \Store as MStore, ungzipResponse, tprintfrom pprint import pprintfrom pymongo import MongoClientimport jsonimport pymongoimport reimport tracebackimport urllibimport urllib2from dtr.storage.Mongo import getDealRankproductsCashbackMap = {"1019749": 5, "1019753":5}ORDERSTATUS = {"PAYMENT_PENDING" : 0,"PAYMENT_FAILED" : 1,"COD_VERIFICATION_PENDING" : 2,"SUBMITTED_FOR_PROCESSING" : 3,"ACCEPTED" : 4,"INVENTORY_LOW" : 5,"REJECTED" : 6,"BILLED" : 7,"PAYMENT_FLAGGED" : 8,"SHIPPED_FROM_WH" : 9,"SHIPPED_TO_LOGST" : 10,"PAYMENT_FLAGGED_DENIED" : 11,"DELIVERY_SUCCESS" : 12,"CANCEL_REQUEST_RECEIVED" : 13,"CANCEL_REQUEST_CONFIRMED" : 14,"CANCELLED_ON_CUSTOMER_REQUEST" : 15,"SHIPPED_TO_DESTINATION_CITY" : 16,"REACHED_DESTINATION_CITY" : 17,"COD_VERIFICATION_FAILED" : 18,"FAILED" : 19,"RTO_IN_TRANSIT" : 20,"RTO_RECEIVED_PRESTINE" : 21,"DOA_PICKUP_REQUEST_RAISED" : 22,"DOA_PICKUP_CONFIRMED" : 23,"DOA_RETURN_IN_TRANSIT" : 24,"DOA_RECEIVED_PRESTINE" : 25,"DOA_CERT_INVALID" : 26,"DOA_CERT_VALID" : 27,"RTO_RESHIPPED" : 28,"DOA_INVALID_RESHIPPED" : 29,"DOA_VALID_RESHIPPED" : 30,"RTO_REFUNDED" : 31,"DOA_VALID_REFUNDED" : 32,"DOA_INVALID_REFUNDED" : 33,"CANCELLED_DUE_TO_LOW_INVENTORY" : 34,"LOW_INV_PO_RAISED" : 35,"LOW_INV_REVERSAL_IN_PROCESS" : 36,"LOW_INV_NOT_AVAILABLE_AT_HOTSPOT" : 37,"LOW_INV_PO_RAISED_TIMEOUT" : 38,"LOW_INV_REVERSAL_TIMEOUT" : 39,"FIRST_DELIVERY_ATTEMPT_MADE" : 40,"CAPTURE_IN_PROCESS" : 41,"DOA_REQUEST_RECEIVED" : 42,"DOA_REQUEST_AUTHORIZED" : 43,"DOA_PICKUP_DENIED" : 44,"DOA_RECEIVED_DAMAGED" : 45,"DOA_LOST_IN_TRANSIT" : 46,"DOA_RESHIPPED_RCVD_DAMAGED" : 47,"DOA_REFUNDED_RCVD_DAMAGED" : 48,"DOA_RESHIPPED_LOST_IN_TRANSIT" : 49,"DOA_REFUNDED_LOST_IN_TRANSIT" : 50,"RTO_RECEIVED_DAMAGED" : 51,"RTO_LOST_IN_TRANSIT" : 52,"RTO_DAMAGED_RESHIPPED" : 53,"RTO_DAMAGED_REFUNDED" : 54,"RTO_LOST_IN_TRANSIT_RESHIPPED" : 55,"RTO_LOST_IN_TRANSIT_REFUNDED" : 56,"RTO_INVENTORY_REVERSED" : 57,"RET_REQUEST_RECEIVED" : 58,"RET_REQUEST_AUTHORIZED" : 59,"RET_PICKUP_REQUEST_RAISED" : 60,"RET_PICKUP_DENIED" : 61,"RET_PICKUP_CONFIRMED" : 62,"RET_RETURN_IN_TRANSIT" : 63,"RET_RECEIVED_PRESTINE" : 64,"RET_RECEIVED_DAMAGED" : 65,"RET_LOST_IN_TRANSIT" : 66,"RET_PRODUCT_USABLE" : 67,"RET_PRODUCT_UNUSABLE" : 68,"RET_PRODUCT_USABLE_RESHIPPED" : 69,"RET_PRODUCT_USABLE_REFUNDED" : 70,"RET_PRODUCT_UNUSABLE_RESHIPPED" : 71,"RET_PRODUCT_UNUSABLE_REFUNDED" : 72,"RET_RESHIPPED_RCVD_DAMAGED" : 73,"RET_REFUNDED_RCVD_DAMAGED" : 74,"RET_RESHIPPED_LOST_IN_TRANSIT" : 75,"RET_REFUNDED_LOST_IN_TRANSIT" : 76,"LOST_IN_TRANSIT" : 77,"LOST_IN_TRANSIT_RESHIPPED" : 78,"LOST_IN_TRANSIT_REFUNDED" : 79,"DELIVERED_AT_STORE" : 80,"RECEIVED_AT_STORE" : 81}headers = {'User-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11','Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8','Accept-Language' : 'en-US,en;q=0.8','Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'}SAHOLIC_ORDER_URL=PythonPropertyReader.getConfig('SAHOLIC_ORDER_URL')SAHOLIC_ORDER_URL_TR=PythonPropertyReader.getConfig('SAHOLIC_ORDER_URL_TR')class Store(MStore):'''This is to map order statuses of our system to order statuses of snapdeal.And our statuses will change accordingly.'''OrderStatusMap = {MStore.ORDER_PLACED : [2,3,4,5,6,7,8,13],MStore.ORDER_DELIVERED : [12],MStore.ORDER_SHIPPED : [9,10,16,17,81,80, 40,20],MStore.ORDER_CANCELLED : [1,6,11,14,15,18,19,31,34,28,53,54,55,56,20,21,77,79, 78]}def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):resp = {}paymentId = int(re.findall(r'=(.*)?',orderSuccessUrl)[0])orderRequest = urllib2.Request(SAHOLIC_ORDER_URL %(paymentId), headers=headers)try:self.userId = userIdconnection = urllib2.urlopen(orderRequest)response = connection.read()connection.close()response = json.loads(response)['response']payment = response['payment']subOrders = []orders = response['orders']items = response['itemsMap']merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)merchantOrder.merchantOrderId = payment['merchantTxnId']merchantOrder.paidAmount = payment['amount']for o in orders:subOrders.append(self.createSubOrder(o, items))merchantOrder.subOrders = subOrdersmerchantOrder.placedOn = merchantOrder.subOrders[0].placedOns = todict(merchantOrder)s['sUserId'] = orders[0]['customer_id']if response.get('payment_option') is not None:s['payment_option'] = response ['payment_option']if self._saveToOrder(s):resp['result'] = 'ORDER_CREATED'else:#Order already createdresp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'except:traceback.print_exc()resp['result'] = 'ORDER_NOT_CREATED'return respdef createSubOrder(self, order, items):lineitem = order['lineitems'][0]item = items[str(lineitem['item_id'])]brand = lineitem.get('brand')modelNumber = lineitem.get('model_number')modelName = lineitem.get('model_name')color = lineitem.get('color')productTitle = brand + (" " + modelName if modelName else "") + (" " + modelNumber if modelNumber is not None else "") + ("(" + color +")" if color else "")amountPaid = order['total_amount']subOrder = SubOrder(productTitle, None, datetime.strftime(datetime.fromtimestamp(order['created_timestamp']/1000),"%d %B %Y"), amountPaid)subOrder.amount = amountPaid - order['gvAmount']subOrder.merchantSubOrderId = str(order['id'])subOrder.orderDetailUrl = "http://m.saholic.com/order/" + subOrder.merchantSubOrderIdsubOrder.quantity = int(lineitem['quantity'])subOrder.estimatedDeliveryDate = datetime.strftime(datetime.fromtimestamp(order['promised_delivery_time']/1000),"%d %B %Y")subOrder.imgUrl = item.get("imgUrl")subOrder.productUrl = "http://m.saholic.com/" + item['url']subOrder.productCode = item['url'].split('-')[-1]subOrder.detailedStatus = order['statusDescription'](cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, amountPaid/subOrder.quantity)if cashbackAmount > 0 and productsCashbackMap.has_key(subOrder.productCode):quantity = productsCashbackMap.get(subOrder.productCode)if subOrder.quantity < quantity:cashbackAmount = 0dealRank = getDealRank(subOrder.productCode, self.store_id, self.userId)subOrder.dealRank = dealRank.get('rank')subOrder.rankDesc = dealRank.get('description')subOrder.maxNlc = dealRank.get('maxNlc')subOrder.minNlc = dealRank.get('minNlc')subOrder.db = dealRank.get('dp')subOrder.itemStatus = dealRank.get('status')cashbackStatus = Store.CB_PENDINGif cashbackAmount <= 0:cashbackStatus = Store.CB_NAsubOrder.cashBackStatus = cashbackStatussubOrder.cashBackAmount = cashbackAmount*subOrder.quantityif percentage > 0:subOrder.cashBackPercentage = percentagereturn subOrderdef _getStatusFromDetailedStatus(self, detailedStatus):print str(detailedStatus)for key, value in Store.OrderStatusMap.iteritems():if detailedStatus in value:return keyprint "Detailed Status need to be mapped", detailedStatusraise ParseException("_getStatusFromDetailedStatus", "Found new order status" + str(detailedStatus))def scrapeStoreOrders(self,):trs = self._getActiveOrders(collectionMap={'merchantOrderId':1, 'sUserId':1, 'userId':1})bulk = self.db.merchantOrder.initialize_ordered_bulk_op()for tr in trs:self.userId = tr['userId']orderRequest = urllib2.Request(SAHOLIC_ORDER_URL_TR %(tr['merchantOrderId'], tr['sUserId']), headers=headers)try:response = urllib2.urlopen(orderRequest).read()print "transaction_id----------", tr['merchantOrderId']response = json.loads(response)['response']items = response['itemsMap']orders = response['orders']closed = TruesplitSubOrdersMap={}ordersToClone = {}for order in orders:#print "orderid---", order['id']orderId = str(order['id'])subOrder = self._isSubOrderActive(tr, orderId)currentQty = int(order['lineitems'][0]['quantity'])if subOrder:if subOrder['closed']:continueelse:quantity = int(subOrder['quantity'])findMap = {"orderId": tr['orderId'], "subOrders.merchantSubOrderId": orderId}updateMap = {}updateMap["subOrders.$.detailedStatus"] = order['statusDescription']status = self._getStatusFromDetailedStatus(ORDERSTATUS[order['status']])closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]updateMap["subOrders.$.status"] = status#Check if splitif quantity != currentQty:updateMap["subOrders.$.quantity"] = currentQtyupdateMap["subOrders.$.amount"] = int((subOrder['amount']*currentQty)/quantity)updateMap["subOrders.$.cashBackAmount"] = int((subOrder['cashBackAmount']*currentQty)/quantity)updateMap["subOrders.$.amountPaid"] = int((subOrder['amountPaid']*currentQty)/quantity)splitSubOrdersMap[order['id']]=subOrderif closedStatus:#if status is closed then change the paybackStatus accordinglyupdateMap["subOrders.$.closed"] = Trueif status == Store.ORDER_DELIVERED:deliveredOn = datetime.strftime(datetime.fromtimestamp(order['delivery_timestamp']/1000),"%A %d %B %Y")updateMap['subOrders.$.deliveredOn'] = deliveredOnif subOrder.get("cashBackStatus") == Store.CB_PENDING:updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVEDelif status == Store.ORDER_CANCELLED:if subOrder.get("cashBackStatus") == Store.CB_PENDING:updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLEDelse:expectedDelivery = datetime.strftime(datetime.fromtimestamp(order['promised_delivery_time']/1000),"%A %d %B %Y")updateMap["subOrders.$.estimatedDeliveryDate"] = expectedDeliveryclosed = Falsebulk.find(findMap).update({'$set' : updateMap})else:if order['originalOrderId']:ordersToClone[order['id']] = orderelse:subOrder = self.createSubOrder(order, items)self.db.merchantOrder.update({"orderId":tr['orderId']},{'$push':{"subOrders":todict(subOrder)}})print "Added new suborder with subOrder Id:", subOrder.merchantSubOrderIdclosed = Falsefor order in ordersToClone.values():originalOrderId = self.getOriginalOrderId(order['id'], ordersToClone, splitSubOrdersMap)subOrderToClone = splitSubOrdersMap[originalOrderId].copy()subOrderToClone["merchantSubOrderId"] = str(order['id'])currentQty = int(order['lineitems'][0]['quantity'])quantity = int(subOrderToClone['quantity'])subOrderToClone["detailedStatus"] = order['statusDescription']status = self._getStatusFromDetailedStatus(ORDERSTATUS[order['status']])closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]print "status---", status, order['status']subOrderToClone["status"] = statussubOrderToClone["quantity"] = currentQtysubOrderToClone["amount"] = int((subOrderToClone['amount']*currentQty)/quantity)subOrderToClone["cashBackAmount"] = int((subOrderToClone['cashBackAmount']*currentQty)/quantity)subOrderToClone["amountPaid"] = int((subOrderToClone['amountPaid']*currentQty)/quantity)if closedStatus:#if status is closed then change the paybackStatus accordinglysubOrderToClone["closed"] = Trueif status == Store.ORDER_DELIVERED:deliveredOn = datetime.strftime(datetime.fromtimestamp(order['delivery_timestamp']/1000),"%A %d %B %Y")subOrderToClone['deliveredOn'] = deliveredOnif subOrderToClone.get("cashBackStatus") == Store.CB_PENDING:subOrderToClone["cashBackStatus"] = Store.CB_APPROVEDelif status == Store.ORDER_CANCELLED:if subOrderToClone.get("cashBackStatus") == Store.CB_PENDING:subOrderToClone["cashBackStatus"] = Store.CB_CANCELLEDelse:expectedDelivery = datetime.strftime(datetime.fromtimestamp(order['promised_delivery_time']/1000),"%A %d %B %Y")subOrderToClone["estimatedDeliveryDate"] = expectedDeliveryclosed = Falseself.db.merchantOrder.update({"orderId":tr['orderId']},{'$push':{"subOrders":subOrderToClone}})bulk.find({"orderId":tr['orderId']}).update({'$set' : {'closed':closed}})except:print "something went wrong for request", orderRequesttraceback.print_exc()try:bulk.execute()except:print "Could not execute bulk"traceback.print_exc()def getOriginalOrderId(self, orderId, ordersToClone, splitSubOrdersMap):if splitSubOrdersMap.has_key(orderId):return orderIdelse:originalOrderId = ordersToClone[orderId]['originalOrderId']return self.getOriginalOrderId(originalOrderId, ordersToClone, splitSubOrdersMap)def _saveToAffiliate(self, offers):if offers is None or len(offers)==0:print "no affiliate have been pushed"returncollection = self.db.snapdealOrderAffiliateInfotry:collection.insert(offers,continue_on_error=True)except pymongo.errors.DuplicateKeyError as e:print e.detailsdef covertToObj(self,offer):offerData = offer['Stat']offer1 = AffiliateInfo(offerData['affiliate_info1'], self.store_id, offerData['conversion_status'], offerData['ad_id'],offerData['datetime'], offerData['payout'], offer['Offer']['name'], offerData['ip'], offerData['conversion_sale_amount'])return offer1def getPostData(token, page = 1, limit= 20, startDate=None, endDate=None):endDate=date.today() + timedelta(days=1)startDate=endDate - timedelta(days=31)parameters = (("page",str(page)),("limit",str(limit)),("fields[]","Stat.offer_id"),("fields[]","Stat.datetime"),("fields[]","Offer.name"),("fields[]","Stat.conversion_status"),("fields[]","Stat.conversion_sale_amount"),("fields[]","Stat.payout"),("fields[]","Stat.ip"),("fields[]","Stat.ad_id"),("fields[]","Stat.affiliate_info1"),("sort[Stat.datetime]","desc"),("filters[Stat.date][conditional]","BETWEEN"),("filters[Stat.date][values][]",startDate.strftime('%Y-%m-%d')),("filters[Stat.date][values][]",endDate.strftime('%Y-%m-%d')),("data_start",startDate.strftime('%Y-%m-%d')),("data_end",endDate.strftime('%Y-%m-%d')),("Method","getConversions"),("NetworkId","jasper"),("SessionToken",token),)#Encode the parametersreturn urllib.urlencode(parameters)def main():store = getStore(4)store.scrapeStoreOrders()#store._isSubOrderActive(8, "5970688907")#store.scrapeAffiliate()#store.parseOrderRawHtml(112345, "subtagId", 1122323, "html", 'http://www.saholic.com/pay-success?paymentId=1798123')#print store.getCashbackAmount('1011378', 500)if __name__ == '__main__':main()def todict(obj, classkey=None):if isinstance(obj, dict):data = {}for (k, v) in obj.items():data[k] = todict(v, classkey)return dataelif hasattr(obj, "_ast"):return todict(obj._ast())elif hasattr(obj, "__iter__"):return [todict(v, classkey) for v in obj]elif hasattr(obj, "__dict__"):data = dict([(key, todict(value, classkey))for key, value in obj.__dict__.iteritems()if not callable(value) and not key.startswith('_')])if classkey is not None and hasattr(obj, "__class__"):data[classkey] = obj.__class__.__name__return dataelse:return obj