Rev 16189 | Rev 17677 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | RSS feed
'''Created on Jan 15, 2015@author: amit'''from BeautifulSoup import BeautifulSoupfrom bson.binary import Binaryfrom datetime import datetime, date, timedeltafrom dtr import mainfrom dtr.config import PythonPropertyReaderfrom dtr.dao import AffiliateInfo, Order, SubOrderfrom dtr.main import getBrowserObject, ScrapeException, getStore, ParseException, \Store as MStore, ungzipResponse, tprintfrom pprint import pprintfrom pymongo import MongoClientimport jsonimport pymongoimport reimport tracebackimport urllibimport urllib2from dtr.storage.Mongo import getDealRankORDERSTATUS = {"PAYMENT_PENDING" : 0,"PAYMENT_FAILED" : 1,"COD_VERIFICATION_PENDING" : 2,"SUBMITTED_FOR_PROCESSING" : 3,"ACCEPTED" : 4,"INVENTORY_LOW" : 5,"REJECTED" : 6,"BILLED" : 7,"PAYMENT_FLAGGED" : 8,"SHIPPED_FROM_WH" : 9,"SHIPPED_TO_LOGST" : 10,"PAYMENT_FLAGGED_DENIED" : 11,"DELIVERY_SUCCESS" : 12,"CANCEL_REQUEST_RECEIVED" : 13,"CANCEL_REQUEST_CONFIRMED" : 14,"CANCELLED_ON_CUSTOMER_REQUEST" : 15,"SHIPPED_TO_DESTINATION_CITY" : 16,"REACHED_DESTINATION_CITY" : 17,"COD_VERIFICATION_FAILED" : 18,"FAILED" : 19,"RTO_IN_TRANSIT" : 20,"RTO_RECEIVED_PRESTINE" : 21,"DOA_PICKUP_REQUEST_RAISED" : 22,"DOA_PICKUP_CONFIRMED" : 23,"DOA_RETURN_IN_TRANSIT" : 24,"DOA_RECEIVED_PRESTINE" : 25,"DOA_CERT_INVALID" : 26,"DOA_CERT_VALID" : 27,"RTO_RESHIPPED" : 28,"DOA_INVALID_RESHIPPED" : 29,"DOA_VALID_RESHIPPED" : 30,"RTO_REFUNDED" : 31,"DOA_VALID_REFUNDED" : 32,"DOA_INVALID_REFUNDED" : 33,"CANCELLED_DUE_TO_LOW_INVENTORY" : 34,"LOW_INV_PO_RAISED" : 35,"LOW_INV_REVERSAL_IN_PROCESS" : 36,"LOW_INV_NOT_AVAILABLE_AT_HOTSPOT" : 37,"LOW_INV_PO_RAISED_TIMEOUT" : 38,"LOW_INV_REVERSAL_TIMEOUT" : 39,"FIRST_DELIVERY_ATTEMPT_MADE" : 40,"CAPTURE_IN_PROCESS" : 41,"DOA_REQUEST_RECEIVED" : 42,"DOA_REQUEST_AUTHORIZED" : 43,"DOA_PICKUP_DENIED" : 44,"DOA_RECEIVED_DAMAGED" : 45,"DOA_LOST_IN_TRANSIT" : 46,"DOA_RESHIPPED_RCVD_DAMAGED" : 47,"DOA_REFUNDED_RCVD_DAMAGED" : 48,"DOA_RESHIPPED_LOST_IN_TRANSIT" : 49,"DOA_REFUNDED_LOST_IN_TRANSIT" : 50,"RTO_RECEIVED_DAMAGED" : 51,"RTO_LOST_IN_TRANSIT" : 52,"RTO_DAMAGED_RESHIPPED" : 53,"RTO_DAMAGED_REFUNDED" : 54,"RTO_LOST_IN_TRANSIT_RESHIPPED" : 55,"RTO_LOST_IN_TRANSIT_REFUNDED" : 56,"RTO_INVENTORY_REVERSED" : 57,"RET_REQUEST_RECEIVED" : 58,"RET_REQUEST_AUTHORIZED" : 59,"RET_PICKUP_REQUEST_RAISED" : 60,"RET_PICKUP_DENIED" : 61,"RET_PICKUP_CONFIRMED" : 62,"RET_RETURN_IN_TRANSIT" : 63,"RET_RECEIVED_PRESTINE" : 64,"RET_RECEIVED_DAMAGED" : 65,"RET_LOST_IN_TRANSIT" : 66,"RET_PRODUCT_USABLE" : 67,"RET_PRODUCT_UNUSABLE" : 68,"RET_PRODUCT_USABLE_RESHIPPED" : 69,"RET_PRODUCT_USABLE_REFUNDED" : 70,"RET_PRODUCT_UNUSABLE_RESHIPPED" : 71,"RET_PRODUCT_UNUSABLE_REFUNDED" : 72,"RET_RESHIPPED_RCVD_DAMAGED" : 73,"RET_REFUNDED_RCVD_DAMAGED" : 74,"RET_RESHIPPED_LOST_IN_TRANSIT" : 75,"RET_REFUNDED_LOST_IN_TRANSIT" : 76,"LOST_IN_TRANSIT" : 77,"LOST_IN_TRANSIT_RESHIPPED" : 78,"LOST_IN_TRANSIT_REFUNDED" : 79,"DELIVERED_AT_STORE" : 80,"RECEIVED_AT_STORE" : 81}headers = {'User-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11','Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8','Accept-Language' : 'en-US,en;q=0.8','Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'}SAHOLIC_ORDER_URL=PythonPropertyReader.getConfig('SAHOLIC_ORDER_URL')SAHOLIC_ORDER_URL_TR=PythonPropertyReader.getConfig('SAHOLIC_ORDER_URL_TR')class Store(MStore):'''This is to map order statuses of our system to order statuses of snapdeal.And our statuses will change accordingly.'''OrderStatusMap = {MStore.ORDER_PLACED : [2,3,4,5,6,7,8,13],MStore.ORDER_DELIVERED : [12],MStore.ORDER_SHIPPED : [9,10,16,17,20,21,81,80],MStore.ORDER_CANCELLED : [1,6,11,14,15,18,19,31,34,28,53,54,55,56]}def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):resp = {}paymentId = int(re.findall(r'=(.*)?',orderSuccessUrl)[0])orderRequest = urllib2.Request(SAHOLIC_ORDER_URL %(paymentId), headers=headers)try:self.userId = userIdconnection = urllib2.urlopen(orderRequest)response = connection.read()connection.close()response = json.loads(response)['response']payment = response['payment']subOrders = []orders = response['orders']items = response['itemsMap']merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)merchantOrder.merchantOrderId = payment['merchantTxnId']merchantOrder.paidAmount = payment['amount']for o in orders:subOrders.append(self.createSubOrder(o, items))merchantOrder.subOrders = subOrdersmerchantOrder.placedOn = merchantOrder.subOrders[0].placedOns = todict(merchantOrder)s['sUserId'] = orders[0]['customer_id']if self._saveToOrder(s):resp['result'] = 'ORDER_CREATED'else:#Order already createdresp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'except:traceback.print_exc()resp['result'] = 'ORDER_NOT_CREATED'return respdef createSubOrder(self, order, items):lineitem = order['lineitems'][0]item = items[str(lineitem['item_id'])]brand = lineitem.get('brand')modelNumber = lineitem.get('model_number')modelName = lineitem.get('model_name')color = lineitem.get('color')productTitle = brand + (" " + modelName if modelName else "") + (" " + modelNumber if modelNumber is not None else "") + ("(" + color +")" if color else "")amountPaid = order['total_amount']- order['gvAmount']subOrder = SubOrder(productTitle, None, datetime.strftime(datetime.fromtimestamp(order['created_timestamp']/1000),"%d %B %Y"), amountPaid)subOrder.merchantSubOrderId = str(order['id'])subOrder.orderDetailUrl = "http://m.saholic.com/order/" + subOrder.merchantSubOrderIdsubOrder.estimatedDeliveryDate = datetime.strftime(datetime.fromtimestamp(order['promised_delivery_time']/1000),"%d %B %Y")subOrder.imgUrl = item['imgUrl']subOrder.productUrl = "http://m.saholic.com/" + item['url']subOrder.productCode = item['url'].split('-')[-1]subOrder.detailedStatus = order['statusDescription'](cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, amountPaid)dealRank = getDealRank(subOrder.productCode, self.store_id, self.userId)subOrder.dealRank = dealRank.get('rank')subOrder.rankDesc = dealRank.get('description')subOrder.maxNlc = dealRank.get('maxNlc')subOrder.minNlc = dealRank.get('minNlc')subOrder.db = dealRank.get('dp')subOrder.itemStatus = dealRank.get('status')cashbackStatus = Store.CB_PENDINGif cashbackAmount <= 0:cashbackStatus = Store.CB_NAsubOrder.cashBackStatus = cashbackStatussubOrder.cashBackAmount = cashbackAmountif percentage > 0:subOrder.cashBackPercentage = percentagereturn subOrderdef _getStatusFromDetailedStatus(self, detailedStatus):for key, value in Store.OrderStatusMap.iteritems():if detailedStatus in value:return keyprint "Detailed Status need to be mapped", detailedStatusraise ParseException("_getStatusFromDetailedStatus", "Found new order status" + str(detailedStatus))def scrapeStoreOrders(self,):trs = self._getActiveOrders(collectionMap={'merchantOrderId':1, 'sUserId':1})bulk = self.db.merchantOrder.initialize_ordered_bulk_op()for tr in trs:try:orderRequest = urllib2.Request(SAHOLIC_ORDER_URL_TR %(tr['merchantOrderId'], tr['sUserId']), headers=headers)response = urllib2.urlopen(orderRequest).read()print responseresponse = json.loads(response)['response']items = response['itemsMap']orders = response['orders']closed = Truefor order in orders:print "orderid---", order['id']orderId = str(order['id'])subOrder = self._isSubOrderActive(tr, orderId)if subOrder:if subOrder['closed']:continueelse:findMap = {"orderId": tr['orderId'], "subOrders.merchantSubOrderId": orderId}updateMap = {}updateMap["subOrders.$.detailedStatus"] = order['statusDescription']status = self._getStatusFromDetailedStatus(ORDERSTATUS[order['status']])closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]print "status---", status, order['status']updateMap["subOrders.$.status"] = statusif closedStatus:#if status is closed then change the paybackStatus accordinglyupdateMap["subOrders.$.closed"] = Trueif status == Store.ORDER_DELIVERED:deliveredOn = datetime.strftime(datetime.fromtimestamp(order['delivery_timestamp']/1000),"%A %d %B %Y")updateMap['subOrders.$.deliveredOn'] = deliveredOnif subOrder.get("cashBackStatus") == Store.CB_PENDING:updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVEDelif status == Store.ORDER_CANCELLED:if subOrder.get("cashBackStatus") == Store.CB_PENDING:updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLEDelse:expectedDelivery = datetime.strftime(datetime.fromtimestamp(order['promised_delivery_time']/1000),"%A %d %B %Y")updateMap["subOrders.$.estimatedDeliveryDate"] = expectedDeliveryclosed = Falsebulk.find(findMap).update({'$set' : updateMap})else:subOrder = self.parseSubOrder(order, items)self.db.merchantOrder.update({"orderId":tr['orderId']},{'$push':{"subOrders":todict(subOrder)}})print "Added new suborder with subOrder Id:", subOrder.merchantSubOrderIdclosed = Falsebulk.find({"orderId":tr['orderId']}).update({'$set' : {'closed':closed}})except:traceback.print_exc()try:bulk.execute()except:traceback.print_exc()def _saveToAffiliate(self, offers):if offers is None or len(offers)==0:print "no affiliate have been pushed"returncollection = self.db.snapdealOrderAffiliateInfotry:collection.insert(offers,continue_on_error=True)except pymongo.errors.DuplicateKeyError as e:print e.detailsdef covertToObj(self,offer):offerData = offer['Stat']offer1 = AffiliateInfo(offerData['affiliate_info1'], self.store_id, offerData['conversion_status'], offerData['ad_id'],offerData['datetime'], offerData['payout'], offer['Offer']['name'], offerData['ip'], offerData['conversion_sale_amount'])return offer1def getPostData(token, page = 1, limit= 20, startDate=None, endDate=None):endDate=date.today() + timedelta(days=1)startDate=endDate - timedelta(days=31)parameters = (("page",str(page)),("limit",str(limit)),("fields[]","Stat.offer_id"),("fields[]","Stat.datetime"),("fields[]","Offer.name"),("fields[]","Stat.conversion_status"),("fields[]","Stat.conversion_sale_amount"),("fields[]","Stat.payout"),("fields[]","Stat.ip"),("fields[]","Stat.ad_id"),("fields[]","Stat.affiliate_info1"),("sort[Stat.datetime]","desc"),("filters[Stat.date][conditional]","BETWEEN"),("filters[Stat.date][values][]",startDate.strftime('%Y-%m-%d')),("filters[Stat.date][values][]",endDate.strftime('%Y-%m-%d')),("data_start",startDate.strftime('%Y-%m-%d')),("data_end",endDate.strftime('%Y-%m-%d')),("Method","getConversions"),("NetworkId","jasper"),("SessionToken",token),)#Encode the parametersreturn urllib.urlencode(parameters)def main():store = getStore(4)#store.scrapeStoreOrders()#store._isSubOrderActive(8, "5970688907")#store.scrapeAffiliate()#store.parseOrderRawHtml(12345, "subtagId", 122323, "html", 'http://www.saholic.com/pay-success?paymentId=1528903')print store.getCashbackAmount('1011378', 500)if __name__ == '__main__':main()def todict(obj, classkey=None):if isinstance(obj, dict):data = {}for (k, v) in obj.items():data[k] = todict(v, classkey)return dataelif hasattr(obj, "_ast"):return todict(obj._ast())elif hasattr(obj, "__iter__"):return [todict(v, classkey) for v in obj]elif hasattr(obj, "__dict__"):data = dict([(key, todict(value, classkey))for key, value in obj.__dict__.iteritems()if not callable(value) and not key.startswith('_')])if classkey is not None and hasattr(obj, "__class__"):data[classkey] = obj.__class__.__name__return dataelse:return obj