| Line 5... |
Line 5... |
| 5 |
'''
|
5 |
'''
|
| 6 |
from bs4 import BeautifulSoup
|
6 |
from bs4 import BeautifulSoup
|
| 7 |
from bson.binary import Binary
|
7 |
from bson.binary import Binary
|
| 8 |
from datetime import datetime, date, timedelta
|
8 |
from datetime import datetime, date, timedelta
|
| 9 |
from dtr import main
|
9 |
from dtr import main
|
| - |
|
10 |
from dtr.api.Service import Orders
|
| 10 |
from dtr.dao import AffiliateInfo, Order, SubOrder
|
11 |
from dtr.dao import AffiliateInfo, Order, SubOrder
|
| 11 |
from dtr.main import getBrowserObject, ScrapeException, getStore, ParseException, \
|
12 |
from dtr.main import getBrowserObject, ScrapeException, getStore, ParseException, \
|
| 12 |
Store as MStore, ungzipResponse, tprint
|
13 |
Store as MStore, ungzipResponse, tprint
|
| 13 |
from dtr.storage import Mongo
|
14 |
from dtr.storage import Mongo
|
| 14 |
from dtr.storage.Mongo import getImgSrc, getDealRank
|
15 |
from dtr.storage.Mongo import getImgSrc, getDealRank
|
| 15 |
from dtr.utils import utils
|
16 |
from dtr.utils import utils
|
| 16 |
from dtr.utils.utils import fetchResponseUsingProxy
|
17 |
from dtr.utils.utils import fetchResponseUsingProxy
|
| - |
|
18 |
from elixir import *
|
| 17 |
from pprint import pprint
|
19 |
from pprint import pprint
|
| 18 |
from pymongo import MongoClient
|
20 |
from pymongo import MongoClient
|
| 19 |
from urlparse import urlparse, parse_qs
|
21 |
from urlparse import urlparse, parse_qs
|
| 20 |
from xlrd import open_workbook
|
22 |
from xlrd import open_workbook
|
| - |
|
23 |
import csv
|
| 21 |
import json
|
24 |
import json
|
| 22 |
import os.path
|
25 |
import os.path
|
| 23 |
import pymongo
|
26 |
import pymongo
|
| 24 |
import re
|
27 |
import re
|
| 25 |
import time
|
28 |
import time
|
| 26 |
import traceback
|
29 |
import traceback
|
| 27 |
import urllib
|
30 |
import urllib
|
| 28 |
import csv
|
31 |
from dtr.storage.DataService import Order_Parse_Info
|
| 29 |
|
32 |
|
| 30 |
USERNAME='profittill2@gmail.com'
|
33 |
USERNAME='profittill2@gmail.com'
|
| 31 |
PASSWORD='spice@2020'
|
34 |
PASSWORD='spice@2020'
|
| 32 |
AFFILIATE_URL='http://affiliate.snapdeal.com/login/'
|
35 |
AFFILIATE_URL='http://affiliate.snapdeal.com/login/'
|
| 33 |
POST_URL='https://api-p03.hasoffers.com/v3/Affiliate_Report.json'
|
36 |
POST_URL='https://api-p03.hasoffers.com/v3/Affiliate_Report.json'
|
| Line 661... |
Line 664... |
| 661 |
# offerData = offer['Stat']
|
664 |
# offerData = offer['Stat']
|
| 662 |
# offer1 = AffiliateInfo(offerData['affiliate_info1'], self.store_id, offerData['conversion_status'], offerData['ad_id'],
|
665 |
# offer1 = AffiliateInfo(offerData['affiliate_info1'], self.store_id, offerData['conversion_status'], offerData['ad_id'],
|
| 663 |
# offerData['datetime'], int(float(offerData['payout'])), offer['Offer']['name'], offerData['ip'], int(float(offerData['conversion_sale_amount'])))
|
666 |
# offerData['datetime'], int(float(offerData['payout'])), offer['Offer']['name'], offerData['ip'], int(float(offerData['conversion_sale_amount'])))
|
| 664 |
# offer1.saleTime = int(time.mktime(datetime.strptime(offer1.saleDate, "%Y-%m-%d %H:%M:%S").timetuple()))
|
667 |
# offer1.saleTime = int(time.mktime(datetime.strptime(offer1.saleDate, "%Y-%m-%d %H:%M:%S").timetuple()))
|
| 665 |
# return offer1
|
668 |
# return offer1
|
| - |
|
669 |
def parseInfo(self,):
|
| - |
|
670 |
from pyquery import PyQuery as pq
|
| - |
|
671 |
orders = session.query(Orders).filter_by(store_id=self.store_id).filter_by(status='ORDER_CREATED').all()
|
| - |
|
672 |
print "total count", len(list(orders))
|
| - |
|
673 |
try:
|
| - |
|
674 |
for order in orders:
|
| - |
|
675 |
try:
|
| - |
|
676 |
doc = pq(order.rawhtml)
|
| - |
|
677 |
a1= " ".join(["" if not div.text else div.text.replace("\t","").replace("\n","").replace(" ", "") for div in pq(doc('article')[-1])('div')])
|
| - |
|
678 |
a2 = ",".join(["" if not div.text else div.text.replace("\t","").replace("\n","").replace(" ", "") for div in pq(doc('article')[-2])('div')])
|
| - |
|
679 |
orderInfo = Order_Parse_Info()
|
| - |
|
680 |
orderInfo.address = a1
|
| - |
|
681 |
all = a2.split(",")
|
| - |
|
682 |
orderInfo.user_id = order.user_id
|
| - |
|
683 |
orderInfo.order_id = order.id
|
| - |
|
684 |
orderInfo.email = all[3].split(":")[1]
|
| - |
|
685 |
orderInfo.name = all[2].split(":")[1]
|
| - |
|
686 |
orderInfo.mobile = all[-1].split(":")[1]
|
| - |
|
687 |
session.commit()
|
| - |
|
688 |
except:
|
| - |
|
689 |
session.rollback()
|
| - |
|
690 |
continue
|
| - |
|
691 |
finally:
|
| - |
|
692 |
session.close()
|
| 666 |
|
693 |
|
| 667 |
def covertToObj(self,offer):
|
694 |
def covertToObj(self,offer):
|
| 668 |
product,category,order_code,quantity,price,sales,commission_rate,commission_earned,datetime1,affiliate_sub_id1,affiliate_sub_id2 = offer
|
695 |
product,category,order_code,quantity,price,sales,commission_rate,commission_earned,datetime1,affiliate_sub_id1,affiliate_sub_id2 = offer
|
| 669 |
offer1 = AffiliateInfo(affiliate_sub_id1, 3, None, None, utils.toTimeStamp(datetime.strptime(datetime1, "%m-%d-%Y %H:%M")),
|
696 |
offer1 = AffiliateInfo(affiliate_sub_id1, 3, None, None, utils.toTimeStamp(datetime.strptime(datetime1, "%m-%d-%Y %H:%M")),
|
| 670 |
0 if "-" else int(float(commission_earned)), None, None, int(float(sales)))
|
697 |
0 if "-" else int(float(commission_earned)), None, None, int(float(sales)))
|
| Line 700... |
Line 727... |
| 700 |
("SessionToken",token),
|
727 |
("SessionToken",token),
|
| 701 |
)
|
728 |
)
|
| 702 |
#Encode the parameters
|
729 |
#Encode the parameters
|
| 703 |
return urllib.urlencode(parameters)
|
730 |
return urllib.urlencode(parameters)
|
| 704 |
|
731 |
|
| - |
|
732 |
|
| 705 |
def main():
|
733 |
def main():
|
| 706 |
#print todict([1,2,"3"])
|
734 |
#print todict([1,2,"3"])
|
| 707 |
store = getStore(3)
|
735 |
store = getStore(3)
|
| 708 |
#store.parseOrderRawHtml(332221, "3232311", 2, "32311243", "https://m.snapdeal.com/orderSummary?code=3fbc8a02a1c4d3c4e906f46886de0464&order=5808451506")
|
736 |
#store.parseOrderRawHtml(332221, "3232311", 2, "32311243", "https://m.snapdeal.com/orderSummary?code=3fbc8a02a1c4d3c4e906f46886de0464&order=5808451506")
|
| 709 |
#store.scrapeStoreOrders()
|
737 |
#store.scrapeStoreOrders()
|
| 710 |
#store._isSubOrderActive(8, "5970688907")
|
738 |
#store._isSubOrderActive(8, "5970688907")
|
| 711 |
store.scrapeAffiliate(datetime(2015,4,1))
|
739 |
#store.scrapeAffiliate(datetime(2015,4,1))
|
| 712 |
#store.scrapeStoreOrders()
|
740 |
#store.scrapeStoreOrders()
|
| - |
|
741 |
store.parseInfo()
|
| 713 |
|
742 |
|
| 714 |
|
743 |
|
| 715 |
|
744 |
|
| 716 |
def todict(obj, classkey=None):
|
745 |
def todict(obj, classkey=None):
|
| 717 |
if isinstance(obj, dict):
|
746 |
if isinstance(obj, dict):
|