Subversion Repositories SmartDukaan

Rev

Rev 12329 | Rev 15492 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 12329 Rev 15484
Line 49... Line 49...
49 
        print e
 49 
        print e
50 
        sys.exit(2)
 50 
        sys.exit(2)
51
51
52 
def fetchDetails(request):
 52 
def fetchDetails(request):
53 
    import threading
 53 
    import threading
54 
    login_url = "https://sellercentral.amazon.in/gp/homepage.html"
 - 
 
55 
    br = login(login_url)
 - 
 
56 
    items = session.query(CompetitorPricing).filter(CompetitorPricing.competitorPricing_requestId==request.requestId).all()
 54 
    items = session.query(CompetitorPricing).filter(CompetitorPricing.competitorPricing_requestId==request.requestId).all()
57 
    print items
 55 
    print items
58 
    snapdeal, flipkart, amazon =[],[],[]
 56 
    snapdeal, flipkart, amazon =[],[],[]
59 
    for item in items:
 57 
    for item in items:
60 
        if item.snapdealScraping:
 58 
        if item.snapdealScraping:
Line 68... Line 66...
68 
    t1.daemon = True
 66 
    t1.daemon = True
69 
    t1.start()
 67 
    t1.start()
70 
    t2 = threading.Thread(target=scrapFlipkart, args = (flipkart,))
 68 
    t2 = threading.Thread(target=scrapFlipkart, args = (flipkart,))
71 
    t2.daemon = True
 69 
    t2.daemon = True
72 
    t2.start()
 70 
    t2.start()
73 
    t3 = threading.Thread(target=scrapAmazon, args = (amazon,br))
 71 
    t3 = threading.Thread(target=scrapAmazon, args = (amazon))
74 
    t3.daemon = True
 72 
    t3.daemon = True
75 
    t3.start()
 73 
    t3.start()
76 
    threads.append(t1)
 74 
    threads.append(t1)
77 
    threads.append(t2)
 75 
    threads.append(t2)
78 
    threads.append(t3)
 76 
    threads.append(t3)
Line 88... Line 86...
88 
    for snapdealItem in snapdealItems:
 86 
    for snapdealItem in snapdealItems:
89 
        sdItem = SnapdealItem.get_by(item_id=snapdealItem.item_id)
 87 
        sdItem = SnapdealItem.get_by(item_id=snapdealItem.item_id)
90 
        if sdItem is None:
 88 
        if sdItem is None:
91 
            continue
 89 
            continue
92 
        try:
 90 
        try:
93 
            url="http://www.snapdeal.com/acors/json/gvbps?supc=%s&catId=91&sort=sellingPrice"%(sdItem.supc)
 91 
            url="http://www.snapdeal.com/acors/json/gvbps?supc=%s&catId=175&sort=sellingPrice"%(sdItem.supc)
94 
            print url
 92 
            print url
95 
            time.sleep(1)
 93 
            time.sleep(1)
96 
            req = urllib2.Request(url)
 94 
            req = urllib2.Request(url)
97 
            response = urllib2.urlopen(req)
 95 
            response = urllib2.urlopen(req)
98 
            json_input = response.read()
 96 
            json_input = response.read()
Line 181... Line 179...
181 
    if session.is_active:
 179 
    if session.is_active:
182 
        print "session is active. closing it."
 180 
        print "session is active. closing it."
183 
        session.close()
 181 
        session.close()
184
182
185 
def scrapAmazon(amazonItems,br):
 183 
def scrapAmazon(amazonItems,br):
186 
    from shop2020.model.v1.catalog.script import AmazonScraper, SellerCentralScraper
 184 
    from shop2020.model.v1.catalog.script import AmazonScraper
- 
 
 185 
    from shop2020.model.v1.catalog.impl.DataService import Amazonlisted
187 
    print "Inside amazonitems ",amazonItems
 186 
    print "Inside amazonitems ",amazonItems
188 
    print "len amazon items ",len(amazonItems)
 187 
    print "len amazon items ",len(amazonItems)
189 
    time.sleep(5)
 188 
    time.sleep(5)
190 
    sc = SellerCentralScraper.SellerCentralScraper()
 - 
 
191 
    scraperAmazon = AmazonScraper.AmazonScraper()
 189 
    amScraper = AmazonScraper.AmazonScraper()
192 
    for amazonItem in amazonItems:
 190 
    for amazonItem in amazonItems:
193 
        skuUrlMfn = "https://sellercentral.amazon.in/myi/search/ProductSummary?keyword=%d" %(amazonItem.item_id)
 191 
        amazon_d_item = Amazonlisted.get_by(itemId=amazonItem.item_id)
194 
        skuUrlFba =  "https://sellercentral.amazon.in/myi/search/ProductSummary?keyword=FBA%d" %(amazonItem.item_id)
 - 
 
195 
        try:
 - 
 
196 
            asin, mfnInventory, mfnPrice= sc.requestSku(br, skuUrlMfn)
 - 
 
197 
            fbaAsin, fbaInventory, fbaPrice= sc.requestSku(br, skuUrlFba)
 - 
 
198 
        except Exception as e:
 192 
        if len(amazon_d_item.asin)==0:
199 
            print e
 - 
 
200 
            print "Unable to fetch details from Seller Central for ",amazonItem.item_id
 193 
            print "No asin found for ",amazonItem.item_id
201 
            continue
 194 
            continue
- 
 
 195 
        saholicUrl = "http://www.amazon.in/gp/offer-listing/%s?m=AF6E3O0VE0X4D&s=merchant-items&ie=UTF8&qid=1433584512&sr=1-1"%(amazon_d_item.asin.strip())
- 
 
 196 
        generalUrl = "http://www.amazon.in/dp/%s"%(amazonItem.item_id.strip())
202 
        try:
 197 
        try:
203 
            if len(asin)==0 and len(fbaAsin)==0:
 - 
 
204 
                print "No asin found for ",amazonItem.item_id
 198 
            saholicPrice = (amScraper.read(saholicUrl, False))[0]
205 
                continue
 199 
        except:
206 
            if len(asin)==0:
 200 
            saholicPrice = 0.0
207 
                asin=fbaAsin
 201 
        try:
208 
            url = "http://www.amazon.in/gp/offer-listing/%s/ref=olp_sort_ps"%(asin)
 202 
            cheapestSeller = (amScraper.read(generalUrl, True))
209 
            scraperAmazon.read(url,True)
 203 
            cheapestSellerPrice = cheapestSeller[0]
210 
            lowestSp,lowestSeller = scraperAmazon.createData()
 204 
            cheapestSellerName = cheapestSeller[1]
211 
            amazonItem.lowestAmazonPrice = lowestSp
 205 
        except:
212 
            amazonItem.ourMfnPrice = float(str(mfnPrice).replace("Rs.","").replace(",",""))
 - 
 
213 
            amazonItem.ourFbaPrice = float(str(fbaPrice).replace("Rs.","").replace(",",""))
 206 
            cheapestSellerPrice = 0.0
214 
            amazonItem.ourMfnInventory = int(mfnInventory)
 207 
            cheapestSellerName = ""
215 
            amazonItem.ourFbaInventory = int(fbaInventory)
 208 
        amazonItem.ourAmazonPrice = saholicPrice
216 
            amazonItem.lowestAmazonPrice = float(lowestSp)
 209 
        amazonItem.lowestAmazonPrice = cheapestSellerPrice
217 
            amazonItem.lowestAmazonSeller = lowestSeller
 210 
        amazonItem.lowestAmazonSeller = cheapestSellerName
218 
 
 - 
 
219 
        except Exception as e:
 - 
 
220 
            print e
 - 
 
221 
            print "Unable to fetch details from Amazon Listing page for ",amazonItem.item_id
 - 
 
222 
            continue
 - 
 
223 
    sc =None
 - 
 
224 
    scraperAmazon = None
 - 
 
225 
    amazonItems[:] =[]
211 
    amazonItems[:] =[]
226
212
227 
def getBrowserObject():
 - 
 
228 
        import cookielib
 - 
 
229 
        br = mechanize.Browser(factory=mechanize.RobustFactory())
 - 
 
230 
        cj = cookielib.LWPCookieJar()
 - 
 
231 
        br.set_cookiejar(cj)
 - 
 
232 
        br.set_handle_equiv(True)
 - 
 
233 
        br.set_handle_redirect(True)
 - 
 
234 
        br.set_handle_referer(True)
 - 
 
235 
        br.set_handle_robots(False)
 - 
 
236 
        br.set_debug_http(False)
 - 
 
237 
        br.set_debug_redirects(False)
 - 
 
238 
        br.set_debug_responses(False)
 - 
 
239
- 
 
240 
        br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
 - 
 
241
- 
 
242 
        br.addheaders = [('User-agent','Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11'),
 - 
 
243 
                         ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'),
 - 
 
244 
                         ('Accept-Encoding', 'gzip,deflate,sdch'),
- 
 
245 
                         ('Accept-Language', 'en-US,en;q=0.8'),
- 
 
246 
                         ('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.3')]
 - 
 
247 
        return br
 - 
 
248 
 
 - 
 
249 
def ungzipResponse(r,b):
 - 
 
250 
    headers = r.info()
 - 
 
251 
    if headers['Content-Encoding']=='gzip':
 - 
 
252 
        import gzip
 - 
 
253 
        print "********************"
 - 
 
254 
        print "Deflating gzip response"
 - 
 
255 
        print "********************"
 - 
 
256 
        gz = gzip.GzipFile(fileobj=r, mode='rb')
 - 
 
257 
        html = gz.read()
 - 
 
258 
        gz.close()
 - 
 
259 
        headers["Content-type"] = "text/html; charset=utf-8"
 - 
 
260 
        r.set_data( html )
 - 
 
261 
        b.set_response(r)
 - 
 
262 
 
 - 
 
263
- 
 
264 
def login(url):
 - 
 
265 
    br = getBrowserObject()
 - 
 
266 
    br.open(url)
 - 
 
267 
    response = br.open(url)
 - 
 
268 
    ungzipResponse(response, br)
 - 
 
269 
    #html = response.read()
 - 
 
270 
    #print html
 - 
 
271 
    br.select_form(name="signinWidget")
 - 
 
272 
    br.form['username'] = "kshitij.sood@saholic.com"
 - 
 
273 
    br.form['password'] = "pioneer"
 - 
 
274 
    response = br.submit()
 - 
 
275 
    print "********************"
 - 
 
276 
    print "Attempting to Login"
 - 
 
277 
    print "********************"
 - 
 
278 
    #ungzipResponse(response, br)
 - 
 
279 
    return br
 - 
 
280 
 
 213 
 
281 
def sendMail(request):
 214 
def sendMail(request):
282 
    import smtplib
 215 
    import smtplib
283 
    from email.mime.text import MIMEText
 216 
    from email.mime.text import MIMEText
284 
    from email.mime.multipart import MIMEMultipart
 217 
    from email.mime.multipart import MIMEMultipart