Rev 264 | Blame | Compare with Previous | Last modification | View Log | RSS feed
'''Created on 12-May-2010@author: gaurav'''from datastore.DataDefinition import * #Phones, init, PhoneItemfrom elixir import *import datetimeclass DataHelper:"""Documentation for class DatahelperThis class contains various methods to access the database tablescrawl_id is used to retain past data for comparison, on each new crawl a new crawl_id is generated"""def __init__(self):#init()passdef initxy(self):"""Documentation for method initxyIt calls a method init() so that when one needs to access the helper methods ofthis class to access database, the database tables are in the scope"""init()#pass'''def add_vendor(self,name,url):try:v = Vendor.query.filter_by(name=name)v = v.filter_by(url=url).one()returnexcept:v = Vendor()v.v_name = namev.v_url = urlsession.commit()def get_all_vendors(self):vi = Vendor.query.all()return videf get_all_phones(self):phones = PhoneItem.query.all()return phonesdef add_new_phone(self, url, name, source):try:pi = PhoneItem.query.filter_by(name=name)pi = pi.filter_by(url=url)pi = pi.filter_by(source=source).one()returnexcept:pi = PhoneItem()pi.name = namepi.url = urlpi.source = sourcesession.commit()def add_mobstoreurl(self,url):try:ai = themobilestoreurls.query.filter_by(url=url).one()returnexcept:ai = themobilestoreurls()ai.url = urlsession.commit()def get_allmobstoreurls(self):ai = themobilestoreurls.query.all()return aidef add_new_mobstorephone(self,name,shown_pr,final_pr):try:ai = themobilestorephones.query.filter_by(name=name)ai = ai.filter_by(shown_price=shown_pr)ai = ai.filter_by(final_price=final_pr).one()returnexcept:ai = themobilestorephones()ai.name = nameai.shown_price = shown_prai.final_price = final_prsession.commit()def get_allmobstorephones(self):ai = themobilestorephones.query.all()return aidef add_pricesbolourl(self,url):try:ai = pricesbolourls.query.filter_by(url=url).one()returnexcept:ai = pricesbolourls()ai.url = urlsession.commit()def get_allpricesbolourl(self):vi = pricesbolourls.query.all()return videf set_all_crawled(self,bval):for ph in self.get_all_phones():ph.is_crawled = bvalsession.commit()def set_crawled(self,url,bval):for ph in self.get_all_phones():if ph.url == url:ph.is_crawled = bvalsession.commit()def add_price(self,url,price):try:p = prices.query.filter_by(price=price)p = p.filter_by(url=url).one()returnexcept:for ph in self.get_all_phones():if ph.url == url:ph.price = pricesession.commit()'''def add_newcrawler(self):"""Documentation for method add_newcrawlerThis method is used to add a new crawlid in crawl's table"""cl = crawl()now = datetime.datetime.now()cl.crawled_date = str(now)session.commit()def get_latestcrawler(self):"""Documentation for method get_latestcrawlerThis method is used to get the latest crawler from the crawl's table"""cl = crawl.query.all()sz = len(cl)cl = crawl.query.filter_by(id=sz).one()return cldef get_latestcrawlerid(self):"""Documentation for method get_latestcrawleridThis method is used to get the latest crawler-id from the crawl's table"""cl = crawl.query.all()sz = len(cl)cl = crawl.query.filter_by(id=sz).one().idreturn cldef add_infiphone(self,name,shown_price,final_price):"""Documentation for method add_infiphoneThis method is used to add a phone in infibeam's table"""try:ai = infibeam_data.query.filter_by(name=name)cid = self.get_latestcrawlerid()ai = ai.filter_by(crawl_id=cid).one()returnexcept:ai = infibeam_data()ai.name = nameai.shown_price = shown_priceai.final_price = final_pricecid = self.get_latestcrawlerid()ai.crawl_id = cidsession.commit()def get_all_infibeam_data(self):"""Documentation for method get_all_infibeam_dataThis method is used to retrieve all the phones in infibeam's table"""cid = self.get_latestcrawlerid()phones = infibeam_data.query.filter_by(crawl_id=cid).all()return phonesdef add_univervendor(self,name,site):"""Documentation for method add_univervendorThis method is used to add a vendor in univercell's table"""try:ai = univercell_data.query.filter_by(name=name)ai = ai.filter_by(site=site)cid = self.get_latestcrawlerid()ai = ai.filter_by(crawl_id=cid).one()returnexcept:ai = univercell_data()ai.v_name = nameai.v_site = sitecid = self.get_latestcrawlerid()ai.crawl_id = cidsession.commit()def get_all_univervendors(self):"""Documentation for method get_all_univervendorThis method is used to retrieve all the vendors in univercell's table"""cid = self.get_latestcrawlerid()vi = univercell_data.query.filter_by(crawl_id=cid).all()return videf add_new_univerphone(self,name,shown_price,final_price):"""Documentation for method add_univerphoneThis method is used to add a phone in univercell's table"""try:ai = univercell_items.query.filter_by(name=name)ai = ai.filter_by(shown_price=shown_price)ai = ai.filter_by(final_price=final_price)cid = self.get_latestcrawlerid()ai = ai.filter_by(crawl_id=cid).one()returnexcept:ai = univercell_items()ai.p_title = nameai.p_shown_price = shown_priceai.p_final_price = final_pricecid = self.get_latestcrawlerid()ai.crawl_id = cidsession.commit()def get_all_univercell_phones(self):"""Documentation for method get_all_univercell_phonesThis method is used to retrieve all the phones in univercell's table"""cid = self.get_latestcrawlerid()phones = univercell_items.query.filter_by(crawl_id=cid).all()return phonesdef add_naaptolurl(self,url):"""Documentation for method add_naaptolurlThis method is used to add a url for phone in naaptol's tableThese are taken from sitemap.xml"""try:n = naaptolurls.query.filter_by(url=url)cid = self.get_latestcrawlerid()n = n.filter_by(crawl_id=cid).one()returnexcept:n = naaptolurls()n.url = urlcid = self.get_latestcrawlerid()n.crawl_id = cidsession.commit()def get_allnaaptolurls(self):"""Documentation for method get_allnaaptolurlsThis method is used to retrieve all the url for phones in naaptol's table"""cid = self.get_latestcrawlerid()ai = naaptolurls.query.filter_by(crawl_id=cid).all()return aidef add_morenaaptolurl(self,url):"""Documentation for method add_morenaaptolurlThis method is used to add a url for phone in naaptol's tableThese are the urls generated by replacing 'price' with 'features'which were redirected"""try:n = morenaaptolurls.query.filter_by(url=url)cid = self.get_latestcrawlerid()n = n.filter_by(crawl_id=cid).one()returnexcept:try:n = morenaaptolurls.query.filter_by(url=url)cid = self.get_latestcrawlerid()n = n.filter_by(crawl_id=cid).one()returnexcept:n = morenaaptolurls()n.url = urlcid = self.get_latestcrawlerid()n.crawl_id = cidsession.commit()def get_allmorenaaptolurls(self):"""Documentation for method get_allmorenaaptolurlsThis method is used to retrieve all the extra urls for phones in naaptol's table"""cid = self.get_latestcrawlerid()ai = morenaaptolurls.query.filter_by(crawl_id=cid).all()return aidef add_new_naaptolphone(self,name,range):"""Documentation for method add_new_naaptolphoneThis method is used to add a phone in naaptol's table"""temp = name.lower()cid = self.get_latestcrawlerid()if temp.find("null") != -1:returnfor n in self.get_allnaaptolphones():if n.name == name:if n.range == range:if n.crawl_id == cid:returnai = naaptolphones()ai.name = nameai.range = rangeai.crawl_id = self.get_latestcrawlerid()session.commit()def get_naaptolphone(self, name, range):"""Documentation for method get_naaptolphoneThis method is used to retrieve a phone in naaptol's tablegiven its name and range"""query = naaptolphones.query.filter_by(name=name)query = query.filter_by(range=range)cid = self.get_latestcrawlerid()query = query.filter_by(crawl_id=cid)return query.one()def get_allnaaptolphones(self):"""Documentation for method get_allnaaptolphonesThis method is used to retrieve all the phone in naaptol's table"""cid = self.get_latestcrawlerid()ai = naaptolphones.query.filter_by(crawl_id=cid).all()return aidef add_new_ntonlinesp(self,nid,name,price):"""Documentation for method add_new_ntonlinespThis method is used to add a online-supplier for a particular phonein naaptol's table"""try:n = ntonlinesp.query.filter_by(nid=nid)cid = self.get_latestcrawlerid()n = n.filter_by(crawl_id=cid)n = n.filter_by(name=name).one()returnexcept:ai = ntonlinesp()ai.nid = nidai.name = nameai.price = pricecid = self.get_latestcrawlerid()ai.crawl_id = cidsession.commit()def get_allntonlinesp(self):"""Documentation for method get_allntonlinespThis method is used to retrieve all the online-suppliers in naaptol's table"""cid = self.get_latestcrawlerid()ai = ntonlinesp.query.filter_by(crawl_id=cid).all()return aidef get_ntonlinespbynid(self,nid):"""Documentation for method get_allntonlinespbynidThis method is used to retrieve all the online-supplier for a particular phonein naaptol's table given the id of phone in naaptolphones"""cid = self.get_latestcrawlerid()ai = ntonlinesp.query.filter_by(crawl_id=cid).all()return aidef add_new_ntofflinesp(self,nid,name,price):"""Documentation for method add_new_ntofflinespThis method is used to add a offline-supplier for a particular phonein naaptol's table"""try:n = ntofflinesp.query.filter_by(nid=nid)cid = self.get_latestcrawlerid()n = n.filter_by(crawl_id=cid)n = n.filter_by(name=name).one()returnexcept:ai = ntofflinesp()ai.nid = nidai.name = nameai.price = pricecid = self.get_latestcrawlerid()ai.crawl_id = cidsession.commit()def get_allntofflinesp(self):"""Documentation for method get_allntofflinespThis method is used to retrieve all the offline-suppliers in naaptol's table"""cid = self.get_latestcrawlerid()ai = ntofflinesp.query.filter_by(crawl_id=cid).all()return aidef get_ntofflinespbynid(self,nid):"""Documentation for method get_allntolinespbynidThis method is used to retrieve all the offline-supplier for a particular phonein naaptol's table given the id of phone in naaptolphones"""ai = ntofflinesp.query.filter_by(nid=nid)cid = self.get_latestcrawlerid()ai = ai.filter_by(crawl_id=cid).all()return aidef add_new_mobstorephone_new(self,name,shown_pr,final_pr,extra_info):"""Documentation for method add_new_mobstorephone_newThis method is used to add a phone in themobilestore's table"""try:ai = themobilestorephones_new.query.filter_by(name=name)cid = self.get_latestcrawlerid()ai = ai.filter_by(crawl_id=cid).one()returnexcept:ai = themobilestorephones_new()ai.name = nameai.shown_price = shown_prai.final_price = final_prai.extra_info = extra_infocid = self.get_latestcrawlerid()ai.crawl_id=cidsession.commit()def get_allmobstorephones_new(self):"""Documentation for method get_allmobstorephone_newThis method is used to retrieve all the phones in themobilestore's table"""cid = self.get_latestcrawlerid()ai = themobilestorephones_new.query.filter_by(crawl_id=cid).all()return aidef add_babuchakurl(self,url,no_pages):"""Documentation for method add_babuchakurlThis method is used to add a url for vendor in babuchak's table"""try:ai = babuchak_urls.query.filter_by(url=url)cid = self.get_latestcrawlerid()ai = ai.filter_by(crawl_id=cid).one()returnexcept:ai = babuchak_urls()ai.url = urlai.no_pages = no_pagescid = self.get_latestcrawlerid()ai.crawl_id=cidsession.commit()def get_allbabuchakurls(self):"""Documentation for method get_allbabuchakurlsThis method is used to retrieve all the vendor-urls in babuchak's table"""cid = self.get_latestcrawlerid()ai = babuchak_urls.query.filter_by(crawl_id=cid).all()return aidef add_babuchakphoneurl(self,url):"""Documentation for method add_babuchakphoneurlThis method is used to add a url for phone in babuchak's table"""try:ai = babuchak_phoneurls.query.filter_by(url=url)cid = self.get_latestcrawlerid()ai = ai.filter_by(crawl_id=cid).one()returnexcept:ai = babuchak_phoneurls()ai.url = urlcid = self.get_latestcrawlerid()ai.crawl_id=cidsession.commit()def get_allbabuchakphoneurls(self):"""Documentation for method get_allbabuchakphoneurlsThis method is used to retrieve all the phone-urls in babuchak's table"""cid = self.get_latestcrawlerid()ai = babuchak_phoneurls.query.filter_by(crawl_id=cid).all()return aidef add_babuchakphone(self,name,shown_price,final_price):"""Documentation for method add_babuchakphoneThis method is used to add a phone in babuchak's table"""try:ai = babuchak_phones.query.filter_by(name=name)cid = self.get_latestcrawlerid()ai = ai.filter_by(crawl_id=cid).one()returnexcept:ai = babuchak_phones()ai.name = nameai.shown_price = shown_priceai.final_price = final_pricecid = self.get_latestcrawlerid()ai.crawl_id=cidsession.commit()def get_allbabuchakphones(self):"""Documentation for method get_allbabuchakphonesThis method is used to retrieve all the phone in babuchak's table"""cid = self.get_latestcrawlerid()ai = babuchak_phones.query.filter_by(crawl_id=cid).all()return aidef add_ipbasic(self,name,site):"""Documentation for method add_ipbasicThis method is used to add a url for phone in indiaplaza's table"""try:ai = indiaplaza_data.query.filter_by(name=name)ai = ai.filter_by(site=site).one()cid = self.get_latestcrawlerid()ai = ai.filter_by(crawl_id=cid).one()returnexcept:ai = indiaplaza_data()ai.v_name = nameai.v_site = sitecid = self.get_latestcrawlerid()ai.crawl_id=cidsession.commit()def get_all_ipbasic(self):"""Documentation for method get_all_ipbasicThis method is used to retrieve all phone-urls in indiaplaza's table"""cid = self.get_latestcrawlerid()vi = indiaplaza_data.query.filter_by(crawl_id=cid).all()return videf add_ipextra(self,name,shown_price,final_price,guarantee,shipinfo):"""Documentation for method add_ipextraThis method is used to add a phone in indiaplaza's table"""try:ai = indiaplaza_items.query.filter_by(p_name=name)ai = ai.filter_by(p_shown_price=shown_price)ai = ai.filter_by(p_final_price=final_price)ai = ai.filter_by(p_guaranteeinfo=guarantee)cid = self.get_latestcrawlerid()ai = ai.filter_by(crawl_id=cid)ai = ai.filter_by(p_shipinfo=shipinfo).one()returnexcept:ai = indiaplaza_items()ai.p_name = nameai.p_shown_price = shown_priceai.p_final_price = final_priceai.p_guaranteeinfo = guaranteeai.p_shipinfo = shipinfocid = self.get_latestcrawlerid()ai.crawl_id=cidsession.commit()def get_all_indiaplaza_phones(self):"""Documentation for method get_all_indiaplaza_phonesThis method is used to retrieve all the phones in indiaplaza's table"""cid = self.get_latestcrawlerid()phones = indiaplaza_items.query.filter_by(crawl_id=cid).all()return phonesdef set_extra_vars(self,var,val,desc):"""Documentation for method set_extra_varsThis method is used to add a variable for a particular supplier.Some suppliers has no fixed count of number of pages on which they have dataso to deal with that this function is used to set variables."""try:tm = extra_vars.query.filter_by(var=var).one()tm.val = valtm.desc = descexcept:tm = extra_vars()tm.var = vartm.val = valtm.desc = descdef get_extra_vars(self,var):"""Documentation for method get_extra_varsThis method is used to retrieve a variable's value given its name"""try:#print "in try"tm = extra_vars.query.filter_by(var=var).one()except:#print "in except"tm = "EMPTY"return tm.valdef add_supplier(self,name,site):"""Documentation for method add_suppliersThis method is used to add a supplier in supplier's table"""for s in self.get_all_suppliers():if s.name == name:now = datetime.datetime.now()s.last_crawled = str(now)session.commit()returnai = suppliers()ai.name = nameai.site = sitenow = datetime.datetime.now()ai.last_crawled = str(now)session.commit()def get_all_suppliers(self):"""Documentation for method get_all_suppliersThis method is used to retrieve all the suppliers from the supplier's table"""sup = suppliers.query.all()return supdef get_suppId(self,name):"""Documentation for method get_suppIdThis method is used to retrieve id of the supplier given his namefrom the supplier's table"""for s in self.get_all_suppliers():if s.name == name:return s.iddef get_supp_byId(self,id):"""Documentation for method get_supp_byIdThis method is used to retrieve a supplier given his idfrom the supplier's table"""for s in self.get_all_suppliers():if s.id == id:return sdef get_supp_byName(self,name):"""Documentation for method get_suppbyNameThis method is used to retrieve a supplier given his namefrom the supplier's table"""supps = []for s in self.get_all_suppliers():if s.name == name:supps.append(s)return suppsdef get_supp_bySite(self,site):"""Documentation for method get_suppbySiteThis method is used to retrieve a supplier given his urlfrom the supplier's table"""supps = []for s in self.get_all_suppliers():if s.site == site:supps.append(s)return suppsdef add_models(self,brand,model):"""Documentation for method add_modelsThis method is used to add a model in model's table"""cid = self.get_latestcrawlerid()for m in self.get_all_models():if m.brand == brand:if m.model == model:if m.crawl_id == cid:returnai = models()ai.brand = brandai.model = modelai.crawl_id = cidsession.commit()def get_all_models(self):"""Documentation for method get_all_modelsThis method is used to retrieve all the model from the model's table"""cid = self.get_latestcrawlerid()mod = models.query.filter_by(crawl_id=cid).all()return moddef get_modId(self,brand,model):"""Documentation for method get_modIdThis method is used to retrieve the id of a modelgiven its name from the model's table"""cid = self.get_latestcrawlerid()for m in self.get_all_models():if m.brand == brand:if m.model == model:if m.crawl_id == cid:return m.iddef get_modbyId(self,id):"""Documentation for method get_modIdThis method is used to retrieve a model given its id from the model's table"""cid = self.get_latestcrawlerid()for m in self.get_all_models():if m.id == id and m.crawl_id == cid:return mdef get_modbyModel(self,model):"""Documentation for method get_modIdThis method is used to retrieve a model given its name from the model's table"""cid = self.get_latestcrawlerid()for m in self.get_all_models():if m.model == model and m.crawl_id == cid:return mdef get_modbyBrand(self,brand):"""Documentation for method get_modIdThis method is used to retrieve a model given its brand-name from the model's table"""cid = self.get_latestcrawlerid()mods = []for m in self.get_all_models():if m.brand == brand and m.crawl_id == cid:mods.append(m)return modsdef add_prices(self,mobile_id,supplier_id,quoted_price,final_price,extra_info):"""Documentation for method add_pricesThis method is used to add all the info for a phone in prices's table"""cid = self.get_latestcrawlerid()for p in self.get_all_prices():if p.mobile_id == mobile_id:if p.supplier_id == supplier_id:if p.extra_info == extra_info and p.crawl_id == cid:returnai = prices()ai.mobile_id = mobile_idai.supplier_id = supplier_idai.quoted_price = quoted_priceai.final_price = final_priceai.extra_info = extra_infoai.crawl_id = cidsession.commit()def get_all_prices(self):"""Documentation for method get_all_pricesThis method is used to retrieve all the info for all the phones in prices's table"""cid = self.get_latestcrawlerid()pr = prices.query.filter_by(crawl_id=cid).all()return prdef get_prbyId(self,id):"""Documentation for method get_prbyIdThis method is used to retrieve all the info for a phone given its id in prices's table"""cid = self.get_latestcrawlerid()for p in self.get_all_prices():if p.id == id and p.crawl_id == cid:return pdef get_prbySid(self,supplier_id):"""Documentation for method get_prbySidThis method is used to retrieve all the info for a phone in prices's tablegiven its supplier-id"""cid = self.get_latestcrawlerid()prc = []for p in self.get_all_prices():if p.supplier_id == supplier_id and p.crawl_id == cid:prc.append(p)return prcdef get_prbyMid(self,mobile_id):"""Documentation for method get_prbySidThis method is used to retrieve all the info for a phone in prices's tablegiven its model-id"""cid = self.get_latestcrawlerid()prc = []for p in self.get_all_prices():if p.mobile_id == mobile_id and p.crawl_id == cid:prc.append(p)return prcdef get_price_by_model(self, model_id, supplier_id):"""Documentation for method get_prbySidThis method is used to retrieve all the info for a phone in prices's tablegiven its model-name"""cid = self.get_latestcrawlerid()query = prices.query.filter_by(mobile_id=model_id)query = query.filter_by(supplier_id=supplier_id)query = query.filter_by(crawl_id=cid)return query.one()def add_gs_info(self,mid,guaranteeinfo,shipinfo):"""Documentation for method add_gs_infoThis method is used to add guarantee and ship info for a phone"""cid = self.get_latestcrawlerid()try:gs = guarantee_info.query.filter_by(mid=mid)gs = gs.filter_by(guaranteeinfo=guaranteeinfo)gs = gs.filter_by(crawl_id=cid)gs = gs.filter_by(shipinfo=shipinfo).one()returnexcept:gs = guarantee_info()gs.mid = midgs.guaranteeinfo = guaranteeinfogs.shipinfo = shipinfogs.crawl_id = cidsession.commit()def get_all_gs_info(self):"""Documentation for method get_all_gs_infoThis method is used to retrieve guarantee and ship info for all the phones"""cid = self.get_latestcrawlerid()gsi = guarantee_info.query.filter_by(crawl_id=cid).all()return gsidef get_gs_bymid(self,mid):"""Documentation for method get_gs_bymidThis method is used to retrieve guarantee and ship info for a phonesgiven its phone-id"""cid = self.get_latestcrawlerid()gsi = guarantee_info.query.filter_by(crawl_id=cid).one()return gsi