Subversion Repositories SmartDukaan

'''

Created on 12-May-2010

@author: gaurav

'''

from elixir import *

'''

class PhoneItem(Entity):

    url = Field(String(1000))

    name = Field(String(50))

    price = Field(Integer)

    source = Field(String(100))

    is_crawled = Field(Boolean)

    phones = ManyToOne("Phones")

    def __repr__(self):

        return "%s %s" %(self.url, self.name)

class Phones(Entity):

    vendor = Field(String(100))

    base_url = Field(String(1000))

    items = OneToMany("PhoneItem")

class Vendor(Entity):

    v_name = Field(String(100))

    v_url = Field(String(1000))

class themobilestoreurls(Entity):

    url = Field(String(1000))

class themobilestorephones(Entity):

    name = Field(String(100))

    shown_price = Field(Integer)

    final_price = Field(Integer)

class pricesbolourls(Entity):

    url = Field(String(1000))

class pricesbolophones(Entity):

    name = Field(String(100))

    shown_price = Field(Integer)

    final_price = Field(Integer)

'''

"""

For each class crawl_id is used to retain past data for comparison, on each new crawl a new crawl_id is generated

"""

class crawl(Entity):

    """

    Documentation for class crawl

    It represents database table for crawl, it stores

    crawled_date = date of crawling

    On each new crawl a new entry is made

    Reason for creating this table is for retaining past data for comparison

    """

    crawled_date = Field(DATE(100))

class infibeam_data(Entity):

    """

    Documentation for class infibeam_data

    It represents database table for infibeam, it stores

    name = name of the phone

    shown_price = price offered by infibeam

    final_price = price which one has to pay, 

    final_price = shown_price + taxes + ship-price

    """

    crawl_id = Field(Integer)

    name = Field(String(100))

    shown_price = Field(Integer)

    final_price = Field(Integer)

class univercell_data(Entity):    

    """

    Documentation for class univercell_data

    It represents database table for univercell, it stores

    v_name = name of the vendor

    v_site = url of the vendor

    """

    crawl_id = Field(Integer)

    v_name = Field(String(100))

    v_site = Field(String(1000))

class univercell_items(Entity):

    """

    Documentation for class univercell_items

    It represents database table for univercell, it stores

    p_title = name of the phone

    p_shown_price = price offered by univercell

    p_final_price = price which one has to pay, 

    p_final_price = p_shown_price + taxes + ship-price

    """    

    crawl_id = Field(Integer)

    p_title = Field(String(100))

    p_shown_price = Field(Integer)

    p_final_price = Field(Integer)

class indiaplaza_data(Entity):

    """

    Documentation for class indiaplaza_data

    It represents database table for indiaplaza, it stores

    v_name = name of the vendor

    v_site = url of the vendor

    """

    crawl_id = Field(Integer)    

    v_name = Field(String(100))

    v_site = Field(String(1000))

class indiaplaza_items(Entity): 

    """

    Documentation for class indiaplaza_items

    It represents database table for indiaplaza, it stores

    p_name = name of the phone

    p_shown_price = price offered by indiaplaza

    p_final_price = price which one has to pay, 

    p_final_price = p_shown_price + taxes + ship-price

    p_guaranteeinfo = duaration of guarantee and whether guarantee is from vendor or manufacturer 

    p_shipinfo = how much time would be taken for shipping

    """

    crawl_id = Field(Integer)

    p_name = Field(String(100))   

    p_shown_price = Field(Integer)

    p_final_price = Field(Integer)

    p_guaranteeinfo = Field(String(100))

    p_shipinfo = Field(String(100))

class themobilestorephones_new(Entity):

    """

    Documentation for class themobilestorephones_new

    It represents database table for themobilestore, it stores

    name = name of the phone

    shown_price = price offered by themobilestore

    final_price = price which one has to pay, 

    final_price = shown_price + taxes + ship-price

    extra_info = whether phone can be bought or not

    """

    crawl_id = Field(Integer)

    name = Field(String(100))

    shown_price = Field(Integer)

    final_price = Field(Integer)

    extra_info = Field(String(1000))

class naaptolurls(Entity):

    """

    Documentation for class naaptolurls

    It represents database table for naaptol, it stores

    url = url of the phones, which we got from sitemap.xml

    """    

    crawl_id = Field(Integer)

    url = Field(String(1000))   

class morenaaptolurls(Entity):

    """

    Documentation for class naaptolurls

    It represents database table for naaptol, it stores

    url = url of the phones, here urls are the ones which are redirected 

    and contained 'price' but before storing 'price' is replaced by 'features'

    """

    crawl_id = Field(Integer)

    url = Field(String(1000)) 

class naaptolphones(Entity):

    """

    Documentation for class naaptolphones

    It represents database table for naaptol, it stores

    name = name of the phone, 

    range = price range for each phone

    range is in one of the 3 forms, i.e

    range = a to b

    range = a

    range = a(approx)

    here a,b are integers

    """      

    crawl_id = Field(Integer)

    name = Field(String(100))

    range = Field(String(100))

class ntonlinesp(Entity):

    """

    Documentation for class ntonlinesp

    It represents database table for naaptol, it stores

    nid = id of the phone in naaptolphones 

    name = name of the onlinesupplier, 

    price = price offered by the supplier for the phone 

    """

    crawl_id = Field(Integer)

    nid = Field(Integer)

    name = Field(String(100))

    price = Field(Integer)

class ntofflinesp(Entity):

    """

    Documentation for class ntofflinesp

    It represents database table for naaptol, it stores

    nid = id of the phone in naaptolphones 

    name = name of the offlinesupplier, 

    price = price offered by the supplier for the phone 

    """

    crawl_id = Field(Integer)

    nid = Field(Integer)

    name = Field(String(100))

    price = Field(Integer)

class babuchak_urls(Entity):

    """

    Documentation for class babuchak_urls

    It represents database table for babuchak, it stores

    url = url for the vendors

    no_pages = number of pages for individual vendor

    """

    crawl_id = Field(Integer)

    url = Field(String(100))

    no_pages = Field(Integer)

class babuchak_phoneurls(Entity):

    """

    Documentation for class babuchak_phoneurls

    It represents database table for babuchak, it stores

    url = url for the individual phones

    """

    crawl_id = Field(Integer)

    url = Field(String(100))

class babuchak_phones(Entity):

    """

    Documentation for class babuchak_phones

    It represents database table for babuchak, it stores

    name = name of the phone

    shown_price = price offered by babuchak

    final_price = price which one has to pay, 

    final_price = shown_price + taxes + ship-price

    """

    crawl_id = Field(Integer)

    name = Field(String(100))

    shown_price = Field(Integer)

    final_price = Field(Integer)

class suppliers(Entity):

    """

    Documentation for class suppliers

    It represents database table for suppliers, it stores

    name = name of the supplier

    site = url of the supplier

    last_crawled = date of the last run for this supplier

    This table spans all the suppliers in our database

    """

    name = Field(String(100))

    site = Field(String(100))

    last_crawled = Field(DATE(100))

class models(Entity):

    """

    Documentation for class models

    It represents database table for models, it stores

    brand = name of the brand for a particular phone

    model = name of the model for a particular phone

    This table spans all the phones-models in our database

    """

    crawl_id = Field(Integer)

    brand = Field(String(100))

    model = Field(String(100))

class prices(Entity):

    """

    Documentation for class prices

    It represents database table for prices, it stores

    supplier_id = id of the supplier who is selling this phone, from suppliers table

    mobile_id = id of the model of this phone, from models table

    quoted_price = price of the phone as offered by the supplier 

    final_price = price one has to pay to buy this phone, i.e

    it includes vat, tax and shippping charges

    extra_info = extra-info about this phone

    This table spans all the phones in our database

    """

    crawl_id = Field(Integer)

    supplier_id = Field(Integer)

    mobile_id = Field(Integer)

    quoted_price = Field(Integer)  

    final_price = Field(Integer)

    extra_info = Field(String(1000))

class guarantee_info(Entity):

    """

    Documentation for class guarantee_info

    It represents database table for guarantee_info, it stores

    mid = id of the phone in models table

    guaranteeinfo = duaration of guarantee and whether guarantee is from vendor or manufacturer 

    shipinfo = how much time would be taken for shipping.

    This table spans all the phones in our database

    """

    crawl_id = Field(Integer)

    mid = Field(Integer)

    guaranteeinfo = Field(String(100))

    shipinfo = Field(String(100))

class extra_vars(Entity):

    """

    Documentation for class extra_vars

    It represents database table for extra_vars, it stores

    var = name of the variable

    val = value of the variable

    desc = description of the variable

    For some suppliers the number of pages to be crawled is not fixed,

    for them variables are created and based on the value of the variable 

    number of the pages to be crawled is determined dynamically

    """

    var = Field(String(100))    

    val = Field(String(100))

    desc = Field(String(1000))

def init():

    """

    Documentation for method init

    Before using all the tables described in this module, one has to call this method

    """

    #metadata.bind = "sqlite:///phones.sqlite"

    metadata.bind = "mysql://root@localhost/phonecrawler"

    metadata.bind.echo = True

    setup_all(True)

    pass

if __name__ == "__main__":

   init()