WebSVN – SmartDukaan – Diff – /prototype/ScrapyDatastore/src/datastore/DataDefinition.py

 @author: gaurav
 '''
 from elixir import *
-#flag = False
+'''
 class PhoneItem(Entity):
     url = Field(String(1000))
     name = Field(String(50))
     price = Field(Integer)
     source = Field(String(100))
 class Vendor(Entity):
     v_name = Field(String(100))
     v_url = Field(String(1000))
+class themobilestoreurls(Entity):
+    url = Field(String(1000))
+class themobilestorephones(Entity):
+    name = Field(String(100))
+    shown_price = Field(Integer)
+    final_price = Field(Integer)
+class pricesbolourls(Entity):
+    url = Field(String(1000))
+class pricesbolophones(Entity):
+    name = Field(String(100))
+    shown_price = Field(Integer)
+    final_price = Field(Integer)
+'''
+"""
+For each class crawl_id is used to retain past data for comparison, on each new crawl a new crawl_id is generated
+"""
+class crawl(Entity):
+    """
+    Documentation for class crawl
+    It represents database table for crawl, it stores
+    crawled_date = date of crawling
+    On each new crawl a new entry is made
+    Reason for creating this table is for retaining past data for comparison
+    """
+    crawled_date = Field(DATE(100))
 class infibeam_data(Entity):
+    """
+    Documentation for class infibeam_data
+    It represents database table for infibeam, it stores
+    name = name of the phone
+    shown_price = price offered by infibeam
+    final_price = price which one has to pay,
+    final_price = shown_price + taxes + ship-price
+    """
+    crawl_id = Field(Integer)
     name = Field(String(100))
     shown_price = Field(Integer)
     final_price = Field(Integer)
 class univercell_data(Entity):
+    """
+    Documentation for class univercell_data
+    It represents database table for univercell, it stores
+    v_name = name of the vendor
+    v_site = url of the vendor
+    """
+    crawl_id = Field(Integer)
     v_name = Field(String(100))
     v_site = Field(String(1000))
 class univercell_items(Entity):
+    """
+    Documentation for class univercell_items
+    It represents database table for univercell, it stores
+    p_title = name of the phone
+    p_shown_price = price offered by univercell
+    p_final_price = price which one has to pay,
+    p_final_price = p_shown_price + taxes + ship-price
+    """
+    crawl_id = Field(Integer)
     p_title = Field(String(100))
     p_shown_price = Field(Integer)
     p_final_price = Field(Integer)
 class indiaplaza_data(Entity):
+    """
+    Documentation for class indiaplaza_data
+    It represents database table for indiaplaza, it stores
+    v_name = name of the vendor
+    v_site = url of the vendor
+    """
+    crawl_id = Field(Integer)
     v_name = Field(String(100))
     v_site = Field(String(1000))
 class indiaplaza_items(Entity):
+    """
+    Documentation for class indiaplaza_items
+    It represents database table for indiaplaza, it stores
+    p_name = name of the phone
+    p_shown_price = price offered by indiaplaza
+    p_final_price = price which one has to pay,
+    p_final_price = p_shown_price + taxes + ship-price
+    p_guaranteeinfo = duaration of guarantee and whether guarantee is from vendor or manufacturer
+    p_shipinfo = how much time would be taken for shipping
+    """
+    crawl_id = Field(Integer)
     p_name = Field(String(100))
     p_shown_price = Field(Integer)
     p_final_price = Field(Integer)
     p_guaranteeinfo = Field(String(100))
     p_shipinfo = Field(String(100))
-class themobilestoreurls(Entity):
-    url = Field(String(1000))
-class themobilestorephones(Entity):
-    name = Field(String(100))
-    shown_price = Field(Integer)
-    final_price = Field(Integer)
 class themobilestorephones_new(Entity):
+    """
+    Documentation for class themobilestorephones_new
+    It represents database table for themobilestore, it stores
+    name = name of the phone
+    shown_price = price offered by themobilestore
+    final_price = price which one has to pay,
+    final_price = shown_price + taxes + ship-price
+    extra_info = whether phone can be bought or not
+    """
+    crawl_id = Field(Integer)
     name = Field(String(100))
     shown_price = Field(Integer)
     final_price = Field(Integer)
     extra_info = Field(String(1000))
-class extra_vars(Entity):
-    var = Field(String(100))
-    val = Field(String(100))
-    desc = Field(String(1000))
-class pricesbolourls(Entity):
-    url = Field(String(1000))
-class pricesbolophones(Entity):
-    name = Field(String(100))
-    shown_price = Field(Integer)
-    final_price = Field(Integer)
 class naaptolurls(Entity):
+    """
+    Documentation for class naaptolurls
+    It represents database table for naaptol, it stores
+    url = url of the phones, which we got from sitemap.xml
+    """
+    crawl_id = Field(Integer)
     url = Field(String(1000))
 class morenaaptolurls(Entity):
+    """
+    Documentation for class naaptolurls
+    It represents database table for naaptol, it stores
+    url = url of the phones, here urls are the ones which are redirected
+    and contained 'price' but before storing 'price' is replaced by 'features'
+    """
+    crawl_id = Field(Integer)
     url = Field(String(1000))
 class naaptolphones(Entity):
+    """
+    Documentation for class naaptolphones
+    It represents database table for naaptol, it stores
+    name = name of the phone,
+    range = price range for each phone
+    range is in one of the 3 forms, i.e
+    range = a to b
+    range = a
+    range = a(approx)
+    here a,b are integers
+    """
+    crawl_id = Field(Integer)
     name = Field(String(100))
     range = Field(String(100))
 class ntonlinesp(Entity):
+    """
+    Documentation for class ntonlinesp
+    It represents database table for naaptol, it stores
+    nid = id of the phone in naaptolphones
+    name = name of the onlinesupplier,
+    price = price offered by the supplier for the phone
+    """
+    crawl_id = Field(Integer)
     nid = Field(Integer)
     name = Field(String(100))
     price = Field(Integer)
 class ntofflinesp(Entity):
+    """
+    Documentation for class ntofflinesp
+    It represents database table for naaptol, it stores
+    nid = id of the phone in naaptolphones
+    name = name of the offlinesupplier,
+    price = price offered by the supplier for the phone
+    """
+    crawl_id = Field(Integer)
     nid = Field(Integer)
     name = Field(String(100))
     price = Field(Integer)
 class babuchak_urls(Entity):
+    """
+    Documentation for class babuchak_urls
+    It represents database table for babuchak, it stores
+    url = url for the vendors
+    no_pages = number of pages for individual vendor
+    """
+    crawl_id = Field(Integer)
     url = Field(String(100))
     no_pages = Field(Integer)
 class babuchak_phoneurls(Entity):
+    """
+    Documentation for class babuchak_phoneurls
+    It represents database table for babuchak, it stores
+    url = url for the individual phones
+    """
+    crawl_id = Field(Integer)
     url = Field(String(100))
 class babuchak_phones(Entity):
+    """
+    Documentation for class babuchak_phones
+    It represents database table for babuchak, it stores
+    name = name of the phone
+    shown_price = price offered by babuchak
+    final_price = price which one has to pay,
+    final_price = shown_price + taxes + ship-price
+    """
+    crawl_id = Field(Integer)
     name = Field(String(100))
     shown_price = Field(Integer)
     final_price = Field(Integer)
 class suppliers(Entity):
+    """
+    Documentation for class suppliers
+    It represents database table for suppliers, it stores
+    name = name of the supplier
+    site = url of the supplier
+    last_crawled = date of the last run for this supplier
+    This table spans all the suppliers in our database
+    """
     name = Field(String(100))
     site = Field(String(100))
     last_crawled = Field(DATE(100))
 class models(Entity):
+    """
+    Documentation for class models
+    It represents database table for models, it stores
+    brand = name of the brand for a particular phone
+    model = name of the model for a particular phone
+    This table spans all the phones-models in our database
+    """
+    crawl_id = Field(Integer)
     brand = Field(String(100))
     model = Field(String(100))
-class guarantee_info(Entity):
-    mid = Field(Integer)
-    guaranteeinfo = Field(String(100))
-    shipinfo = Field(String(100))
 class prices(Entity):
+    """
+    Documentation for class prices
+    It represents database table for prices, it stores
+    supplier_id = id of the supplier who is selling this phone, from suppliers table
+    mobile_id = id of the model of this phone, from models table
+    quoted_price = price of the phone as offered by the supplier
+    final_price = price one has to pay to buy this phone, i.e
+    it includes vat, tax and shippping charges
+    extra_info = extra-info about this phone
+    This table spans all the phones in our database
+    """
+    crawl_id = Field(Integer)
     supplier_id = Field(Integer)
     mobile_id = Field(Integer)
     quoted_price = Field(Integer)
     final_price = Field(Integer)
     extra_info = Field(String(1000))
+class guarantee_info(Entity):
+    """
+    Documentation for class guarantee_info
+    It represents database table for guarantee_info, it stores
+    mid = id of the phone in models table
+    guaranteeinfo = duaration of guarantee and whether guarantee is from vendor or manufacturer
+    shipinfo = how much time would be taken for shipping.
+    This table spans all the phones in our database
+    """
+    crawl_id = Field(Integer)
+    mid = Field(Integer)
+    guaranteeinfo = Field(String(100))
+    shipinfo = Field(String(100))
+class extra_vars(Entity):
+    """
+    Documentation for class extra_vars
+    It represents database table for extra_vars, it stores
+    var = name of the variable
+    val = value of the variable
+    desc = description of the variable
+    For some suppliers the number of pages to be crawled is not fixed,
+    for them variables are created and based on the value of the variable
+    number of the pages to be crawled is determined dynamically
+    """
+    var = Field(String(100))
+    val = Field(String(100))
+    desc = Field(String(1000))
 def init():
+    """
+    Documentation for method init
+    Before using all the tables described in this module, one has to call this method
+    """
     #metadata.bind = "sqlite:///phones.sqlite"
     metadata.bind = "mysql://root@localhost/phonecrawler"
     metadata.bind.echo = True
     setup_all(True)
     pass

Subversion Repositories SmartDukaan

(root)/prototype/ScrapyDatastore/src/datastore/DataDefinition.py – Rev 238 → 264