Subversion Repositories SmartDukaan

Rev

Rev 14398 | Rev 14460 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 14398 Rev 14412
Line 4... Line 4...
4 
@author: amit
 4 
@author: amit
5 
'''
 5 
'''
6 
from BeautifulSoup import BeautifulSoup
 6 
from BeautifulSoup import BeautifulSoup
7 
from datetime import datetime, date, timedelta
 7 
from datetime import datetime, date, timedelta
8 
from dtr import main
 8 
from dtr import main
9 
from dtr.dao import AffiliateInfo, Order, SubOrder
 9 
from dtr.dao import AffiliateInfo, Order, SubOrder, FlipkartAffiliateInfo
10 
from dtr.main import getBrowserObject, getStore, ParseException, ungzipResponse, \
 10 
from dtr.main import getBrowserObject, getStore, ParseException, ungzipResponse, \
11 
    Store as MStore, sourceMap, tprint
 11 
    Store as MStore, sourceMap, tprint
12 
from pprint import pprint
 12 
from pprint import pprint
13 
from pymongo.mongo_client import MongoClient
 13 
from pymongo.mongo_client import MongoClient
14 
import hashlib
 14 
import hashlib
15 
import importlib
 15 
import importlib
16 
import json
 16 
import json
17 
import mechanize
 17 
import mechanize
- 
 
 18 
import pymongo
18 
import re
 19 
import re
19 
import traceback
 20 
import traceback
20 
import urllib
 21 
import urllib
21 
 
 22 
 
22 
USERNAME='saholic1@gmail.com'
 23 
USERNAME='saholic1@gmail.com'
Line 137... Line 138...
137 
                tprint("Could not update " + str(order['orderId']))
 138 
                tprint("Could not update " + str(order['orderId']))
138 
                traceback.print_exc()
 139 
                traceback.print_exc()
139 
    def scrapeAffiliate(self, startDate=None, endDate=None):
 140 
    def scrapeAffiliate(self, startDate=None, endDate=None):
140 
        #get all yesterday's affiliate in pending or cancelled state and update to system
 141 
        #get all yesterday's affiliate in pending or cancelled state and update to system
141 
        if datetime.now().hour == 1:
 142 
        if datetime.now().hour == 1:
142 
            yesterdate = date.today() - timedelta(1)
 - 
 
143 
            syesterdate = yesterdate.strftime('%Y-%m-%d')
- 
 
144 
            br = getBrowserObject()
 143 
            br = getBrowserObject()
145 
            br.open(AFFILIATE_URL)
 144 
            br.open(AFFILIATE_URL)
146 
            response = br.response()  # copy
 145 
            response = br.response()  # copy
147 
            token =  re.findall('window.__FK = "(.*?)"', ungzipResponse(response), re.IGNORECASE)[0]
 146 
            token =  re.findall('window.__FK = "(.*?)"', ungzipResponse(response), re.IGNORECASE)[0]
148 
            data = {'__FK':token,
 147 
            data = {'__FK':token,
149 
                    'email':'saholic1@gmail.com',
 148 
                    'email':'saholic1@gmail.com',
150 
                    'password':'e8aacf6fc1e3998186a4a8e56e428f66'
 149 
                    'password':'e8aacf6fc1e3998186a4a8e56e428f66'
151 
            }
 150 
            }
152 
            br.open(AFFILIATE_LOGIN_URL, urllib.urlencode(data))
 151 
            br.open(AFFILIATE_LOGIN_URL, urllib.urlencode(data))
- 
 
 152 
            offers = []
- 
 
 153 
            for delta in 1,2,3,4,5, 6,7:
- 
 
 154 
                yester5date = date.today() - timedelta(delta)
- 
 
 155 
                syester5date = yester5date.strftime('%Y-%m-%d')
153 
            for status in [AFF_STATUS_PENDING, AFF_STATUS_CANCELLED, AFF_STATUS_DISAPPROVED]:
 156 
                for status in [AFF_STATUS_PENDING, AFF_STATUS_CANCELLED, AFF_STATUS_DISAPPROVED]:
154 
                br.open(AFF_REPORT_URL % (status, syesterdate, syesterdate))
 157 
                    br.open(AFF_REPORT_URL % (status, syester5date, syester5date))
155 
                page = ungzipResponse(br.response())
 158 
                    page = ungzipResponse(br.response())
156 
                soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
 159 
                    soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
- 
 
 160 
                    soup.table
157 
                try:
 161 
                    try:
158 
                    tableElement = soup.findAll('table', {'class':'report-table fixtable'})[1]
 162 
                        tableElement = soup.findAll('table', {'class':'report-table fixtable'})[1]
159 
                except:
 163 
                    except:
160 
                    continue
 164 
                        continue
161 
                trElements = tableElement.findAll('tr')
 165 
                    trElements = tableElement.findAll('tr')
162 
                trElements.pop(0)
 166 
                    trElements.pop(0)
163 
                for trElement in trElements:
 167 
                    for trElement in trElements:
164 
                    tdElements = trElement.findAll('td')
 168 
                        tdElements = trElement.findAll('td')
165 
                    productCode = re.findall(r'pid=(.*)$', tdElements[0].find('a')['href'])[0]
 169 
                        productCode = re.findall(r'pid=(.*)$', tdElements[0].find('a')['href'])[0]
166 
                    quantity =int(tdElements[3].text)
 170 
                        quantity =int(tdElements[3].text)
167 
                    price = int(re.findall(r'\d+', tdElements[2].text)[0])
 171 
                        price = int(float(tdElements[2].text.strip().replace(",","")))
168 
                    payOut = int(re.findall(r'\d+', tdElements[6].text)[0])
 172 
                        payOut = int(float(tdElements[6].text.strip().replace(",","")))
- 
 
 173 
                        saleAmount = int(float(tdElements[4].text.strip().replace(",","")))
169 
                    subTagId = tdElements[7].text
 174 
                        subTagId = tdElements[7].text.strip()
- 
 
 175 
                        updateMap={}
170 
                    affiliateInfo = AffiliateInfo(subTagId, self.store_id, status, None, syesterdate, payOut, None, None, price*quantity)
 176 
                        affiliateInfo = FlipkartAffiliateInfo(subTagId, syester5date, productCode, price, quantity, saleAmount, payOut, status)
171 
                    print todict(affiliateInfo)
 177 
                        offers.append(todict(affiliateInfo))
172 
                    updateMap = {}
 - 
 
173 
                    updateMap['subOrders.$.unitPrice'] = price
 178 
                        #updateMap['subOrders.$.unitPrice'] = price
174 
                    updateMap['subOrders.$.cashBackAmount'], updateMap['subOrders.$.cashBacPercentage'] = self.getCashbackAmount(productCode, price)
 179 
                        #updateMap['subOrders.$.cashBackAmount'], updateMap['subOrders.$.cashBacPercentage'] = self.getCashbackAmount(productCode, price)
175 
                    self._updateOrdersPayBackStatus({'subTagId':subTagId, 'subOrders.productCode':productCode}, updateMap)
 180 
                        self._updateOrdersPayBackStatus({'subTagId':subTagId, 'subOrders.productCode':productCode}, updateMap)
- 
 
 181 
            print offers
- 
 
 182 
            self._saveToAffiliate(offers)
176
183
177
184
- 
 
 185 
    def _saveToAffiliate(self, offers):
- 
 
 186 
        if offers is None or len(offers)==0:
- 
 
 187 
            print "no affiliate have been pushed"
- 
 
 188 
            return
- 
 
 189 
        collection = self.db.flipkartOrderAffiliateInfo
- 
 
 190 
        try:
- 
 
 191 
            collection.insert(offers,continue_on_error=True)
- 
 
 192 
        except pymongo.errors.DuplicateKeyError as e:
- 
 
 193 
            print e.details
- 
 
 194
- 
 
 195
178 
    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
 196 
    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
179 
        resp= {}
 197 
        resp= {}
180 
        try:
 198 
        try:
181 
            br = getBrowserObject()
 199 
            br = getBrowserObject()
182 
            url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]
 200 
            url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]
Line 360... Line 378...
360 
    print(m.digest())
 378 
    print(m.digest())
361 
 
 379 
 
362 
def main():
 380 
def main():
363
381
364 
    store = getStore(2)
 382 
    store = getStore(2)
365 
    #store.scrapeAffiliate()
 383 
    store.scrapeAffiliate()
366 
    store.parseOrderRawHtml(123469, "SHA21423034609", 122324,  "html", 'https://m.flipkart.com/order_details?reference_id=OD3022229107123666&token=13ace95c4dc4e6bc372529a330e6b656&src=or&pr=1')
 384 
    #store.scrapeStoreOrders(123469, "SHA21423034609", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0022423903274462&token=8c09a43ad48d1cd4f73be1f0c3c00d39&src=or&pr=1')
367 
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3016502908102575&token=0db4c692bacbfbfc158b52358ac9e91e&src=or&pr=1')
 385 
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3016502908102575&token=0db4c692bacbfbfc158b52358ac9e91e&src=or&pr=1')
368 
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3018701137253850&token=f7402ddcf2b63b37cc6bc528cc115d2f&src=or&pr=1')
 386 
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3018701137253850&token=f7402ddcf2b63b37cc6bc528cc115d2f&src=or&pr=1')
369 
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019279584515727&token=7d85d8c24d36b5a1efc8008634390c7e&src=or&pr=1')
 387 
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019279584515727&token=7d85d8c24d36b5a1efc8008634390c7e&src=or&pr=1')
370 
    #store.flipkartOrderTracking(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019279584515727&token=7d85d8c24d36b5a1efc8008634390c7e&src=or&pr=1')
 388 
    #store.flipkartOrderTracking(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019279584515727&token=7d85d8c24d36b5a1efc8008634390c7e&src=or&pr=1')
371 
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019365336126533&token=dbce2bd4dc4023295b436a7d3c7986c9&src=or&pr=1')
 389 
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019365336126533&token=dbce2bd4dc4023295b436a7d3c7986c9&src=or&pr=1')