WebSVN – SmartDukaan – Diff – /prototype/ScrapyDatastore/src/datastore/DataAccessor.py

 from elixir import *
 import datetime
 class DataHelper:
+    """
+    Documentation for class Datahelper
+    This class contains various methods to access the database tables
+    crawl_id is used to retain past data for comparison, on each new crawl a new crawl_id is generated
+    """
     def __init__(self):
         #init()
         pass
     def initxy(self):
+        """
+        Documentation for method initxy
+        It calls a method init() so that when one needs to access the helper methods of
+        this class to access database, the database tables are in the scope
+        """
         init()
         #pass
+    '''
     def add_vendor(self,name,url):
         try:
             v = Vendor.query.filter_by(name=name)
             v = v.filter_by(url=url).one()
             return
         except:
             v = Vendor()
             v.v_name = name
             v.v_url = url
             session.commit()
-    def set_all_crawled(self,bval):
-        for ph in self.get_all_phones():
-            ph.is_crawled = bval
-    def set_crawled(self,url,bval):
-        for ph in self.get_all_phones():
-            if ph.url == url:
-                ph.is_crawled = bval
-        session.commit()
-    def get_all_phones(self):
-        phones = PhoneItem.query.all()
-        return phones
     def get_all_vendors(self):
         vi = Vendor.query.all()
         return vi
+    def get_all_phones(self):
+        phones = PhoneItem.query.all()
+        return phones
     def add_new_phone(self, url, name, source):
         try:
             pi = PhoneItem.query.filter_by(name=name)
             pi = pi.filter_by(url=url)
             pi.name = name
             pi.url = url
             pi.source = source
             session.commit()
+    def add_mobstoreurl(self,url):
+        try:
+            ai = themobilestoreurls.query.filter_by(url=url).one()
+            return
+        except:
+            ai = themobilestoreurls()
+            ai.url = url
+            session.commit()
+    def get_allmobstoreurls(self):
+        ai = themobilestoreurls.query.all()
+        return ai
+    def add_new_mobstorephone(self,name,shown_pr,final_pr):
+        try:
+            ai = themobilestorephones.query.filter_by(name=name)
+            ai = ai.filter_by(shown_price=shown_pr)
+            ai = ai.filter_by(final_price=final_pr).one()
+            return
+        except:
+            ai = themobilestorephones()
+            ai.name = name
+            ai.shown_price = shown_pr
+            ai.final_price = final_pr
+            session.commit()
+    def get_allmobstorephones(self):
+        ai = themobilestorephones.query.all()
+        return ai
+    def add_pricesbolourl(self,url):
+        try:
+            ai = pricesbolourls.query.filter_by(url=url).one()
+            return
+        except:
+            ai = pricesbolourls()
+            ai.url = url
+            session.commit()
+    def get_allpricesbolourl(self):
+        vi = pricesbolourls.query.all()
+        return vi
+    def set_all_crawled(self,bval):
+        for ph in self.get_all_phones():
+            ph.is_crawled = bval
+        session.commit()
+    def set_crawled(self,url,bval):
+        for ph in self.get_all_phones():
+            if ph.url == url:
+                ph.is_crawled = bval
+        session.commit()
     def add_price(self,url,price):
         try:
             p = prices.query.filter_by(price=price)
             p = p.filter_by(url=url).one()
             return
         except:
             for ph in self.get_all_phones():
                 if ph.url == url:
                     ph.price = price
+            session.commit()
+    '''
+    def add_newcrawler(self):
+        """
+        Documentation for method add_newcrawler
+        This method is used to add a new crawlid in crawl's table
+        """
+        cl = crawl()
+        now = datetime.datetime.now()
+        cl.crawled_date = str(now)
-            session.commit()
+        session.commit()
+    def get_latestcrawler(self):
+        """
+        Documentation for method get_latestcrawler
+        This method is used to get the latest crawler from the crawl's table
+        """
+        cl = crawl.query.all()
+        sz = len(cl)
+        cl = crawl.query.filter_by(id=sz).one()
+        return cl
+    def get_latestcrawlerid(self):
+        """
+        Documentation for method get_latestcrawlerid
+        This method is used to get the latest crawler-id from the crawl's table
+        """
+        cl = crawl.query.all()
+        sz = len(cl)
+        cl = crawl.query.filter_by(id=sz).one().id
+        return cl
     def add_infiphone(self,name,shown_price,final_price):
+        """
+        Documentation for method add_infiphone
+        This method is used to add a phone in infibeam's table
+        """
         try:
-            ai = infibeam_data.query.filter_by(name=name).one()
+            ai = infibeam_data.query.filter_by(name=name)
+            cid = self.get_latestcrawlerid()
+            ai = ai.filter_by(crawl_id=cid).one()
             return
         except:
             ai = infibeam_data()
             ai.name = name
             ai.shown_price = shown_price
             ai.final_price = final_price
+            cid = self.get_latestcrawlerid()
+            ai.crawl_id = cid
             session.commit()
+    def get_all_infibeam_data(self):
+        """
+        Documentation for method get_all_infibeam_data
+        This method is used to retrieve all the phones in infibeam's table
+        """
+        cid = self.get_latestcrawlerid()
+        phones = infibeam_data.query.filter_by(crawl_id=cid)
+        return phones
     def add_univervendor(self,name,site):
+        """
+        Documentation for method add_univervendor
+        This method is used to add a vendor in univercell's table
+        """
         try:
             ai = univercell_data.query.filter_by(name=name)
-            ai = ai.filter_by(site=site).one()
+            ai = ai.filter_by(site=site)
+            cid = self.get_latestcrawlerid()
+            ai = ai.filter_by(crawl_id=cid).one()
             return
         except:
             ai = univercell_data()
             ai.v_name = name
             ai.v_site = site
+            cid = self.get_latestcrawlerid()
+            ai.crawl_id = cid
             session.commit()
     def get_all_univervendors(self):
+        """
+        Documentation for method get_all_univervendor
+        This method is used to retrieve all the vendors in univercell's table
+        """
+        cid = self.get_latestcrawlerid()
-        vi = univercell_data.query.all()
+        vi = univercell_data.query.filter_by(crawl_id=cid)
         return vi
     def add_new_univerphone(self,name,shown_price,final_price):
+        """
+        Documentation for method add_univerphone
+        This method is used to add a phone in univercell's table
+        """
         try:
             ai = univercell_items.query.filter_by(name=name)
             ai = ai.filter_by(shown_price=shown_price)
-            ai = ai.filter_by(final_price=final_price).one()
+            ai = ai.filter_by(final_price=final_price)
+            cid = self.get_latestcrawlerid()
+            ai = ai.filter_by(crawl_id=cid).one()
             return
         except:
             ai = univercell_items()
             ai.p_title = name
             ai.p_shown_price = shown_price
             ai.p_final_price = final_price
+            cid = self.get_latestcrawlerid()
+            ai.crawl_id = cid
             session.commit()
-    def add_ipbasic(self,name,site):
+    def get_all_univercell_phones(self):
-        try:
+        """
-            ai = indiaplaza_data.query.filter_by(name=name)
+        Documentation for method get_all_univercell_phones
-            ai = ai.filter_by(site=site).one()
+        This method is used to retrieve all the phones in univercell's table
-            return
-        except:
-            ai = indiaplaza_data()
-            ai.v_name = name
-            ai.v_site = site
-            session.commit()
+        """
-    def add_mobstoreurl(self,url):
+        cid = self.get_latestcrawlerid()
-        try:
-            ai = themobilestoreurls.query.filter_by(url=url).one()
+        phones = univercell_items.query.filter_by(crawl_id=cid)
-            return
+        return phones
-        except:
-            ai = themobilestoreurls()
-            ai.url = url
-            session.commit()
-    def add_pricesbolourl(self,url):
-        try:
-            ai = pricesbolourls.query.filter_by(url=url).one()
-            return
-        except:
-            ai = pricesbolourls()
-            ai.url = url
-            session.commit()
-    def get_allpricesbolourl(self):
-        vi = pricesbolourls.query.all()
-        return vi
     def add_naaptolurl(self,url):
+        """
+        Documentation for method add_naaptolurl
+        This method is used to add a url for phone in naaptol's table
+        These are taken from sitemap.xml
+        """
         try:
-            n = naaptolurls.query.filter_by(url=url).one()
+            n = naaptolurls.query.filter_by(url=url)
+            cid = self.get_latestcrawlerid()
+            n = n.filter_by(crawl_id=cid).one()
             return
         except:
-            ai = naaptolurls()
+            n = naaptolurls()
-            ai.url = url
+            n.url = url
+            cid = self.get_latestcrawlerid()
+            n.crawl_id = cid
             session.commit()
+    def get_allnaaptolurls(self):
+        """
+        Documentation for method get_allnaaptolurls
+        This method is used to retrieve all the url for phones in naaptol's table
+        """
+        cid = self.get_latestcrawlerid()
+        ai = naaptolurls.query.filter_by(crawl_id=cid)
+        return ai
     def add_morenaaptolurl(self,url):
+        """
+        Documentation for method add_morenaaptolurl
+        This method is used to add a url for phone in naaptol's table
+        These are the urls generated by replacing 'price' with 'features'
+        which were redirected
+        """
         try:
-            n = morenaaptolurls.query.filter_by(url=url).one()
+            n = morenaaptolurls.query.filter_by(url=url)
+            cid = self.get_latestcrawlerid()
+            n = n.filter_by(crawl_id=cid).one()
             return
         except:
             try:
-                n = naaptolurls.query.filter_by(url=url).one()
+                n = morenaaptolurls.query.filter_by(url=url)
+                cid = self.get_latestcrawlerid()
+                n = n.filter_by(crawl_id=cid).one()
                 return
             except:
-                ai = morenaaptolurls()
+                n = morenaaptolurls()
-                ai.url = url
+                n.url = url
+                cid = self.get_latestcrawlerid()
+                n.crawl_id = cid
                 session.commit()
-    def get_allmobstoreurls(self):
-        ai = themobilestoreurls.query.all()
-        return ai
-    def get_allnaaptolurls(self):
-        ai = naaptolurls.query.all()
-        return ai
     def get_allmorenaaptolurls(self):
+        """
+        Documentation for method get_allmorenaaptolurls
+        This method is used to retrieve all the extra urls for phones in naaptol's table
+        """
+        cid = self.get_latestcrawlerid()
-        ai = morenaaptolurls.query.all()
+        ai = morenaaptolurls.query.filter_by(crawl_id=cid)
         return ai
-    def add_new_mobstorephone(self,name,shown_pr,final_pr):
-        try:
-            ai = themobilestorephones.query.filter_by(name=name)
-            ai = ai.filter_by(shown_price=shown_pr)
-            ai = ai.filter_by(final_price=final_pr).one()
-            return
-        except:
-            ai = themobilestorephones()
-            ai.name = name
-            ai.shown_price = shown_pr
-            ai.final_price = final_pr
-            session.commit()
-    def add_new_mobstorephone_new(self,name,shown_pr,final_pr,extra_info):
-        try:
-            ai = themobilestorephones_new.query.filter_by(name=name).one()
-            return
-        except:
-            ai = themobilestorephones_new()
-            ai.name = name
-            ai.shown_price = shown_pr
-            ai.final_price = final_pr
-            ai.extra_info = extra_info
-            session.commit()
     def add_new_naaptolphone(self,name,range):
+        """
+        Documentation for method add_new_naaptolphone
+        This method is used to add a phone in naaptol's table
+        """
         temp = name.lower()
+        cid = self.get_latestcrawlerid()
         if temp.find("null") != -1:
             return
         for n in self.get_allnaaptolphones():
             if n.name == name:
                 if n.range == range:
+                    if n.crawl_id == cid:
                         return
         ai = naaptolphones()
         ai.name = name
         ai.range = range
+        ai.crawl_id = self.get_latestcrawlerid()
         session.commit()
+    def get_naaptolphone(self, name, range):
+        """
+        Documentation for method get_naaptolphone
+        This method is used to retrieve a phone in naaptol's table
+        given its name and range
+        """
+        query = naaptolphones.query.filter_by(name=name)
+        query = query.filter_by(range=range)
+        cid = self.get_latestcrawlerid()
+        query = query.filter_by(crawl_id=cid)
+        return query.one()
+    def get_allnaaptolphones(self):
+        """
+        Documentation for method get_allnaaptolphones
+        This method is used to retrieve all the phone in naaptol's table
+        """
+        cid = self.get_latestcrawlerid()
+        ai = naaptolphones.query.filter_by(crawl_id=cid)
+        return ai
     def add_new_ntonlinesp(self,nid,name,price):
+        """
+        Documentation for method add_new_ntonlinesp
+        This method is used to add a online-supplier for a particular phone
+        in naaptol's table
+        """
         try:
             n = ntonlinesp.query.filter_by(nid=nid)
+            cid = self.get_latestcrawlerid()
+            n = n.filter_by(crawl_id=cid)
             n = n.filter_by(name=name).one()
             return
         except:
             ai = ntonlinesp()
             ai.nid = nid
             ai.name = name
             ai.price = price
+            cid = self.get_latestcrawlerid()
+            ai.crawl_id = cid
             session.commit()
+    def get_allntonlinesp(self):
+        """
+        Documentation for method get_allntonlinesp
+        This method is used to retrieve all the online-suppliers in naaptol's table
+        """
+        cid = self.get_latestcrawlerid()
+        ai = ntonlinesp.query.filter_by(crawl_id=cid)
+        return ai
+    def get_ntonlinespbynid(self,nid):
+        """
+        Documentation for method get_allntonlinespbynid
+        This method is used to retrieve all the online-supplier for a particular phone
+        in naaptol's table given the id of phone in naaptolphones
+        """
+        cid = self.get_latestcrawlerid()
+        ai = ntonlinesp.query.filter_by(crawl_id=cid)
+        return ai
     def add_new_ntofflinesp(self,nid,name,price):
+        """
+        Documentation for method add_new_ntofflinesp
+        This method is used to add a offline-supplier for a particular phone
+        in naaptol's table
+        """
         try:
             n = ntofflinesp.query.filter_by(nid=nid)
+            cid = self.get_latestcrawlerid()
+            n = n.filter_by(crawl_id=cid)
             n = n.filter_by(name=name).one()
             return
         except:
             ai = ntofflinesp()
             ai.nid = nid
             ai.name = name
             ai.price = price
+            cid = self.get_latestcrawlerid()
+            ai.crawl_id = cid
             session.commit()
-    def get_naaptolphone(self, name, range):
-        query = naaptolphones.query.filter_by(name=name)
-        query = query.filter_by(range=range)
-        return query.one()
-    def get_allmobstorephones(self):
-        ai = themobilestorephones.query.all()
-        return ai
-    def get_allmobstorephones_new(self):
-        ai = themobilestorephones_new.query.all()
-        return ai
-    def get_allnaaptolphones(self):
-        ai = naaptolphones.query.all()
-        return ai
-    def get_allntonlinesp(self):
-        ai = ntonlinesp.query.all()
-        return ai
     def get_allntofflinesp(self):
+        """
-        ai = ntofflinesp.query.all()
+        Documentation for method get_allntofflinesp
-        return ai
+        This method is used to retrieve all the offline-suppliers in naaptol's table
+        """
-    def get_ntonlinespbynid(self,nid):
+        cid = self.get_latestcrawlerid()
-        ai = ntonlinesp.query.filter_by(nid=nid)
+        ai = ntofflinesp.query.filter_by(crawl_id=cid)
         return ai
     def get_ntofflinespbynid(self,nid):
+        """
+        Documentation for method get_allntolinespbynid
+        This method is used to retrieve all the offline-supplier for a particular phone
+        in naaptol's table given the id of phone in naaptolphones
+        """
         ai = ntofflinesp.query.filter_by(nid=nid)
+        cid = self.get_latestcrawlerid()
+        ai = ai.filter_by(crawl_id=cid)
+        return ai
+    def add_new_mobstorephone_new(self,name,shown_pr,final_pr,extra_info):
+        """
+        Documentation for method add_new_mobstorephone_new
+        This method is used to add a phone in themobilestore's table
+        """
+        try:
+            ai = themobilestorephones_new.query.filter_by(name=name)
+            cid = self.get_latestcrawlerid()
+            ai = ai.filter_by(crawl_id=cid).one()
+            return
+        except:
+            ai = themobilestorephones_new()
+            ai.name = name
+            ai.shown_price = shown_pr
+            ai.final_price = final_pr
+            ai.extra_info = extra_info
+            cid = self.get_latestcrawlerid()
+            ai.crawl_id=cid
+            session.commit()
+    def get_allmobstorephones_new(self):
+        """
+        Documentation for method get_allmobstorephone_new
+        This method is used to retrieve all the phones in themobilestore's table
+        """
+        cid = self.get_latestcrawlerid()
+        ai = themobilestorephones_new.query.filter_by(crawl_id=cid)
         return ai
     def add_babuchakurl(self,url,no_pages):
+        """
+        Documentation for method add_babuchakurl
+        This method is used to add a url for vendor in babuchak's table
+        """
         try:
-            ai = babuchak_urls.query.filter_by(url=url).one()
+            ai = babuchak_urls.query.filter_by(url=url)
+            cid = self.get_latestcrawlerid()
+            ai = ai.filter_by(crawl_id=cid).one()
             return
         except:
             ai = babuchak_urls()
             ai.url = url
             ai.no_pages = no_pages
+            cid = self.get_latestcrawlerid()
+            ai.crawl_id=cid
             session.commit()
     def get_allbabuchakurls(self):
+        """
+        Documentation for method get_allbabuchakurls
+        This method is used to retrieve all the vendor-urls in babuchak's table
+        """
+        cid = self.get_latestcrawlerid()
-        ai = babuchak_urls.query.all()
+        ai = babuchak_urls.query.filter_by(crawl_id=cid)
         return ai
     def add_babuchakphoneurl(self,url):
+        """
+        Documentation for method add_babuchakphoneurl
+        This method is used to add a url for phone in babuchak's table
+        """
         try:
-            ai = babuchak_phoneurls.query.filter_by(url=url).one()
+            ai = babuchak_phoneurls.query.filter_by(url=url)
+            cid = self.get_latestcrawlerid()
+            ai = ai.filter_by(crawl_id=cid).one()
             return
         except:
             ai = babuchak_phoneurls()
             ai.url = url
+            cid = self.get_latestcrawlerid()
+            ai.crawl_id=cid
             session.commit()
     def get_allbabuchakphoneurls(self):
+        """
+        Documentation for method get_allbabuchakphoneurls
+        This method is used to retrieve all the phone-urls in babuchak's table
+        """
+        cid = self.get_latestcrawlerid()
-        ai = babuchak_phoneurls.query.all()
+        ai = babuchak_phoneurls.query.filter_by(crawl_id=cid)
         return ai
     def add_babuchakphone(self,name,shown_price,final_price):
+        """
+        Documentation for method add_babuchakphone
+        This method is used to add a phone in babuchak's table
+        """
         try:
-            ai = babuchak_phones.query.filter_by(name=name).one()
+            ai = babuchak_phones.query.filter_by(name=name)
+            cid = self.get_latestcrawlerid()
+            ai = ai.filter_by(crawl_id=cid).one()
             return
         except:
             ai = babuchak_phones()
             ai.name = name
             ai.shown_price = shown_price
             ai.final_price = final_price
+            cid = self.get_latestcrawlerid()
+            ai.crawl_id=cid
             session.commit()
     def get_allbabuchakphones(self):
+        """
+        Documentation for method get_allbabuchakphones
+        This method is used to retrieve all the phone in babuchak's table
+        """
+        cid = self.get_latestcrawlerid()
-        ai = babuchak_phones.query.all()
+        ai = babuchak_phones.query.filter_by(crawl_id=cid)
         return ai
+    def add_ipbasic(self,name,site):
+        """
+        Documentation for method add_ipbasic
+        This method is used to add a url for phone in indiaplaza's table
+        """
+        try:
+            ai = indiaplaza_data.query.filter_by(name=name)
+            ai = ai.filter_by(site=site).one()
+            cid = self.get_latestcrawlerid()
+            ai = ai.filter_by(crawl_id=cid).one()
+            return
+        except:
+            ai = indiaplaza_data()
+            ai.v_name = name
+            ai.v_site = site
+            cid = self.get_latestcrawlerid()
+            ai.crawl_id=cid
+            session.commit()
     def get_all_ipbasic(self):
+        """
+        Documentation for method get_all_ipbasic
+        This method is used to retrieve all phone-urls in indiaplaza's table
+        """
+        cid = self.get_latestcrawlerid()
-        vi = indiaplaza_data.query.all()
+        vi = indiaplaza_data.query.filter_by(crawl_id=cid)
         return vi
     def add_ipextra(self,name,shown_price,final_price,guarantee,shipinfo):
+        """
+        Documentation for method add_ipextra
+        This method is used to add a phone in indiaplaza's table
+        """
         try:
             ai = indiaplaza_items.query.filter_by(p_name=name)
             ai = ai.filter_by(p_shown_price=shown_price)
             ai = ai.filter_by(p_final_price=final_price)
             ai = ai.filter_by(p_guaranteeinfo=guarantee)
+            cid = self.get_latestcrawlerid()
+            ai = ai.filter_by(crawl_id=cid)
             ai = ai.filter_by(p_shipinfo=shipinfo).one()
             return
         except:
             ai = indiaplaza_items()
             ai.p_name = name
             ai.p_shown_price = shown_price
             ai.p_final_price = final_price
             ai.p_guaranteeinfo = guarantee
             ai.p_shipinfo = shipinfo
+            cid = self.get_latestcrawlerid()
+            ai.crawl_id=cid
             session.commit()
-    def get_all_infibeam_data(self):
-        phones = infibeam_data.query.all()
-        return phones
     def get_all_indiaplaza_phones(self):
+        """
-        phones = indiaplaza_items.query.all()
+        Documentation for method get_all_indiaplaza_phones
-        return phones
+        This method is used to retrieve all the phones in indiaplaza's table
+        """
-    def get_all_univercell_phones(self):
+        cid = self.get_latestcrawlerid()
-        phones = univercell_items.query.all()
+        phones = indiaplaza_items.query.filter_by(crawl_id=cid)
         return phones
+    def set_extra_vars(self,var,val,desc):
+        """
+        Documentation for method set_extra_vars
+        This method is used to add a variable for a particular supplier.
+        Some suppliers has no fixed count of number of pages on which they have data
+        so to deal with that this function is used to set variables.
+        """
+        try:
+            tm = extra_vars.query.filter_by(var=var).one()
+            tm.val = val
+            tm.desc = desc
+        except:
+            tm = extra_vars()
+            tm.var = var
+            tm.val = val
+            tm.desc = desc
+    def get_extra_vars(self,var):
+        """
+        Documentation for method get_extra_vars
+        This method is used to retrieve a variable's value given its name
+        """
+        try:
+            #print "in try"
+            tm = extra_vars.query.filter_by(var=var).one()
+        except:
+            #print "in except"
+            tm = "EMPTY"
+        return tm.val
-    def get_infibeam_csv(self):
+    def add_supplier(self,name,site):
+        """
+        Documentation for method add_suppliers
+        This method is used to add a supplier in supplier's table
+        """
-        phones = self.get_all_infibeam_phones()
+        for s in self.get_all_suppliers():
+            if s.name == name:
+                now = datetime.datetime.now()
+                s.last_crawled = str(now)
+                session.commit()
+                return
-        print phones
+        ai = suppliers()
+        ai.name = name
+        ai.site = site
+        now = datetime.datetime.now()
+        ai.last_crawled = str(now)
+        session.commit()
     def get_all_suppliers(self):
+        """
+        Documentation for method get_all_suppliers
+        This method is used to retrieve all the suppliers from the supplier's table
+        """
         sup = suppliers.query.all()
         return sup
-    def get_all_models(self):
-        mod = models.query.all()
-        return mod
-    def get_all_prices(self):
-        pr = prices.query.all()
-        return pr
     def get_suppId(self,name):
+        """
+        Documentation for method get_suppId
+        This method is used to retrieve id of the supplier given his name
+        from the supplier's table
+        """
         for s in self.get_all_suppliers():
             if s.name == name:
                 return s.id
     def get_supp_byId(self,id):
+        """
+        Documentation for method get_supp_byId
+        This method is used to retrieve a supplier given his id
+        from the supplier's table
+        """
         for s in self.get_all_suppliers():
             if s.id == id:
                 return s
     def get_supp_byName(self,name):
+        """
+        Documentation for method get_suppbyName
+        This method is used to retrieve a supplier given his name
+        from the supplier's table
+        """
         supps = []
         for s in self.get_all_suppliers():
             if s.name == name:
                 supps.append(s)
         return supps
     def get_supp_bySite(self,site):
+        """
+        Documentation for method get_suppbySite
+        This method is used to retrieve a supplier given his url
+        from the supplier's table
+        """
         supps = []
         for s in self.get_all_suppliers():
             if s.site == site:
                 supps.append(s)
         return supps
+    def add_models(self,brand,model):
+        """
+        Documentation for method add_models
+        This method is used to add a model in model's table
+        """
+        cid = self.get_latestcrawlerid()
+        for m in self.get_all_models():
+            if m.brand == brand:
+                if m.model == model:
+                    if m.crawl_id == cid:
+                        return
+        ai = models()
+        ai.brand = brand
+        ai.model = model
+        ai.crawl_id = cid
+        session.commit()
+    def get_all_models(self):
+        """
+        Documentation for method get_all_models
+        This method is used to retrieve all the model from the model's table
+        """
+        cid = self.get_latestcrawlerid()
+        mod = models.query.filter_by(crawl_id=cid)
+        return mod
     def get_modId(self,brand,model):
+        """
+        Documentation for method get_modId
+        This method is used to retrieve the id of a model
+        given its name from the model's table
+        """
+        cid = self.get_latestcrawlerid()
         for m in self.get_all_models():
             if m.brand == brand:
                 if m.model == model:
+                    if m.crawl_id == cid:
-                    return m.id
+                        return m.id
     def get_modbyId(self,id):
+        """
+        Documentation for method get_modId
+        This method is used to retrieve a model given its id from the model's table
+        """
+        cid = self.get_latestcrawlerid()
         for m in self.get_all_models():
-            if m.id == id:
+            if m.id == id and m.crawl_id == cid:
                 return m
     def get_modbyModel(self,model):
+        """
+        Documentation for method get_modId
+        This method is used to retrieve a model given its name from the model's table
+        """
+        cid = self.get_latestcrawlerid()
         for m in self.get_all_models():
-            if m.model == model:
+            if m.model == model and m.crawl_id == cid:
                 return m
     def get_modbyBrand(self,brand):
+        """
+        Documentation for method get_modId
+        This method is used to retrieve a model given its brand-name from the model's table
+        """
+        cid = self.get_latestcrawlerid()
         mods = []
         for m in self.get_all_models():
-            if m.brand == brand:
+            if m.brand == brand and m.crawl_id == cid:
                 mods.append(m)
         return mods
+    def add_prices(self,mobile_id,supplier_id,quoted_price,final_price,extra_info):
+        """
+        Documentation for method add_prices
+        This method is used to add all the info for a phone in prices's table
+        """
+        cid = self.get_latestcrawlerid()
+        for p in self.get_all_prices():
+            if p.mobile_id == mobile_id:
+                if p.supplier_id == supplier_id:
+                    if p.extra_info == extra_info and p.crawl_id == cid:
+                        return
+        ai = prices()
+        ai.mobile_id = mobile_id
+        ai.supplier_id = supplier_id
+        ai.quoted_price = quoted_price
+        ai.final_price = final_price
+        ai.extra_info = extra_info
+        ai.crawl_id = cid
+        session.commit()
+    def get_all_prices(self):
+        """
+        Documentation for method get_all_prices
+        This method is used to retrieve all the info for all the phones in prices's table
+        """
+        cid = self.get_latestcrawlerid()
+        pr = prices.query.filter_by(crawl_id=cid)
+        return pr
     def get_prbyId(self,id):
+        """
+        Documentation for method get_prbyId
+        This method is used to retrieve all the info for a phone given its id in prices's table
+        """
+        cid = self.get_latestcrawlerid()
         for p in self.get_all_prices():
-            if p.id == id:
+            if p.id == id and p.crawl_id == cid:
                 return p
     def get_prbySid(self,supplier_id):
+        """
+        Documentation for method get_prbySid
+        This method is used to retrieve all the info for a phone in prices's table
+        given its supplier-id
+        """
+        cid = self.get_latestcrawlerid()
         prc = []
         for p in self.get_all_prices():
-            if p.supplier_id == supplier_id:
+            if p.supplier_id == supplier_id and p.crawl_id == cid:
                 prc.append(p)
         return prc
     def get_prbyMid(self,mobile_id):
+        """
+        Documentation for method get_prbySid
+        This method is used to retrieve all the info for a phone in prices's table
+        given its model-id
+        """
+        cid = self.get_latestcrawlerid()
         prc = []
         for p in self.get_all_prices():
-            if p.mobile_id == mobile_id:
+            if p.mobile_id == mobile_id and p.crawl_id == cid:
                 prc.append(p)
         return prc
-    def add_supplier(self,name,site):
-        for s in self.get_all_suppliers():
-            if s.name == name:
-                now = datetime.datetime.now()
-                s.last_crawled = str(now)
-                session.commit()
-                return
-        ai = suppliers()
-        ai.name = name
-        ai.site = site
-        now = datetime.datetime.now()
-        ai.last_crawled = str(now)
-        session.commit()
-    def add_models(self,brand,model):
-        for m in self.get_all_models():
-            if m.brand == brand:
-                if m.model == model:
-                    return
-        ai = models()
-        ai.brand = brand
-        ai.model = model
-        session.commit()
-    def add_prices(self,mobile_id,supplier_id,quoted_price,final_price,extra_info):
-        for p in self.get_all_prices():
-            if p.mobile_id == mobile_id:
-                if p.supplier_id == supplier_id:
-                    if p.extra_info == extra_info:
-                        return
-        ai = prices()
-        ai.mobile_id = mobile_id
-        ai.supplier_id = supplier_id
-        ai.quoted_price = quoted_price
-        ai.final_price = final_price
-        ai.extra_info = extra_info
-        session.commit()
     def get_price_by_model(self, model_id, supplier_id):
+        """
+        Documentation for method get_prbySid
+        This method is used to retrieve all the info for a phone in prices's table
+        given its model-name
+        """
+        cid = self.get_latestcrawlerid()
         query = prices.query.filter_by(mobile_id=model_id)
         query = query.filter_by(supplier_id=supplier_id)
+        query = query.filter_by(crawl_id=cid)
         return query.one()
     def add_gs_info(self,mid,guaranteeinfo,shipinfo):
+        """
+        Documentation for method add_gs_info
+        This method is used to add guarantee and ship info for a phone
+        """
+        cid = self.get_latestcrawlerid()
         try:
             gs = guarantee_info.query.filter_by(mid=mid)
             gs = gs.filter_by(guaranteeinfo=guaranteeinfo)
+            gs = gs.filter_by(crawl_id=cid)
             gs = gs.filter_by(shipinfo=shipinfo).one()
             return
         except:
             gs = guarantee_info()
             gs.mid = mid
             gs.guaranteeinfo = guaranteeinfo
             gs.shipinfo = shipinfo
+            gs.crawl_id = cid
             session.commit()
     def get_all_gs_info(self):
+        """
+        Documentation for method get_all_gs_info
+        This method is used to retrieve guarantee and ship info for all the phones
+        """
+        cid = self.get_latestcrawlerid()
-        gsi = guarantee_info.query.all()
+        gsi = guarantee_info.query.filter_by(crawl_id=cid)
         return gsi
     def get_gs_bymid(self,mid):
-        gsi = guarantee_info.query.filter_by(mid=mid).one()
-        return gsi
-    def set_extra_vars(self,var,val,desc):
-        try:
-            tm = extra_vars.query.filter_by(var=var).one()
-            tm.val = val
-            tm.desc = desc
-        except:
-            tm = extra_vars()
-            tm.var = var
-            tm.val = val
-            tm.desc = desc
-    def get_extra_vars(self,var):
-        try:
-            #print "in try"
-            tm = extra_vars.query.filter_by(var=var).one()
-        except:
-            #print "in except"
-            tm = "EMPTY"
-        return tm.val
-if __name__ == "__main__":
-    datastore = DataHelper()
-    datastore.get_infibeam_csv()
         """
+        Documentation for method get_gs_bymid
+        This method is used to retrieve guarantee and ship info for a phones
+        given its phone-id
+        """
+        cid = self.get_latestcrawlerid()
+        gsi = guarantee_info.query.filter_by(crawl_id=cid).one()
+        return gsi

Subversion Repositories SmartDukaan

(root)/prototype/ScrapyDatastore/src/datastore/DataAccessor.py – Rev 238 → 264