Subversion Repositories SmartDukaan

package in.shop2020.web;

import java.io.IOException;

import java.util.ArrayList;

import java.util.Date;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import javax.servlet.ServletException;

import javax.servlet.http.HttpServlet;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;

import org.json.JSONArray;

import org.json.JSONException;

import org.json.JSONObject;

import com.google.appengine.api.datastore.DatastoreService;

import com.google.appengine.api.datastore.DatastoreServiceFactory;

import com.google.appengine.api.datastore.Entity;

import com.google.appengine.api.datastore.FetchOptions;

import com.google.appengine.api.datastore.Key;

import com.google.appengine.api.datastore.KeyFactory;

import com.google.appengine.api.datastore.PreparedQuery;

import com.google.appengine.api.datastore.Query;

import com.google.appengine.api.datastore.Query.FilterOperator;

import com.google.appengine.api.datastore.Query.SortDirection;

public class PriceComparisonServlet extends HttpServlet {

	private static final long serialVersionUID = 1L;

	private final String ENTITY_KIND_JOB = "ScrapeJob";

	private final String ENTITY_KIND_PHONE_PRICE = "PhonePrice";

	private DatastoreService ds = DatastoreServiceFactory.getDatastoreService();

	public enum JobStatus {

		UNASSIGNED(0), ASSIGNED(1), COMPLETED(2);

		private int code;

		private JobStatus(int code)	{

			this.code = code;

		}

		public int getCode()	{

			return this.code;

		}

	}

	private Entity createScrapeJob(String url, String source)	{

		Entity scrapeJob = new Entity(ENTITY_KIND_JOB);

		scrapeJob.setProperty("url", url);

		scrapeJob.setProperty("source", source);

		scrapeJob.setProperty("status", JobStatus.UNASSIGNED.getCode());

		scrapeJob.setProperty("assigneeId", null);

		scrapeJob.setProperty("enqueuedOn", null);

		return scrapeJob;

	}

	private void assignScrapeJob(Entity scrapeJob, String assigneeId)	{

		System.out.println("Assignee Id: " + assigneeId);

		scrapeJob.setProperty("status", JobStatus.ASSIGNED.getCode());

		scrapeJob.setProperty("assigneeId", assigneeId);

		ds.put(scrapeJob);

	}

	private void enqueueScrapeJob(Entity scrapeJob)	{

		scrapeJob.setProperty("enqueuedOn", new Date());

		ds.put(scrapeJob);

	}

	private Entity assignJobToScraper(String assigneeId, String lastScrapedSource)	{

		Query query = new Query(ENTITY_KIND_JOB);

		query.addFilter("status", FilterOperator.EQUAL, JobStatus.UNASSIGNED.getCode());

		query.addSort("enqueuedOn", SortDirection.ASCENDING);

		PreparedQuery pq = ds.prepare(query);

		FetchOptions fo = FetchOptions.Builder.withLimit(1);

		Entity job;

		if (pq.countEntities(fo) > 0)	{

			job = pq.asList(fo).get(0);

			if (assigneeId == null)	{

				System.out.println("Assignee Id is null");

				String newAssigneeId = Long.toString(job.getKey().getId());

				System.out.println("New Job Id & Assignee Id: " + newAssigneeId);

				assignScrapeJob(job, newAssigneeId);

			} else	{

				System.out.println("Assignee Id is " + assigneeId);

				assignScrapeJob(job, assigneeId);

			}

		} else	{

			job = null;

		}

		System.out.println("Job assigned:" + job);

		return job;

	}

	private Map<String, String> toMap(Entity job)	{

		Map<String, String> jobMap = new HashMap<String, String>();

		if (job != null)	{

			jobMap.put("id", Long.toString(job.getKey().getId()));

			jobMap.put("source", (String) job.getProperty("source"));

			jobMap.put("assigneeId", (String) job.getProperty("assigneeId"));

			jobMap.put("url", (String) job.getProperty("url"));

		} else	{

			jobMap.put("id", null);

			jobMap.put("source", null);

			jobMap.put("assigneeId", null);

			jobMap.put("url", null);

		}

		return jobMap;

	}

	private int getTimeToWait()	{

		return 2 + (int) (Math.random() * 2);

	}

	private String getPhoneNameKey(String phoneName, String source)	{

		int bracketStartsAt = phoneName.indexOf('(');

		if(bracketStartsAt > -1)	{

			phoneName = phoneName.substring(0, bracketStartsAt);

		}

		String phoneNameKey = phoneName.replaceAll("[^a-zA-Z0-9]", "");

		return phoneNameKey.toLowerCase();

	}

	private void removeJobFromQueue(String jobId)	{

		Key k = KeyFactory.createKey(ENTITY_KIND_JOB, Long.parseLong(jobId));

		ds.delete(k);

	}

	private void savePhonePrices(JSONArray phonePrices, String source)	{

		System.out.println("Saving Price Data");

		List<Entity> priceEntities = new ArrayList<Entity>();

		for (int i = 0; i < phonePrices.length(); i ++)	{

			try {

				JSONObject phone = phonePrices.getJSONObject(i);

				Entity priceEntity = new Entity(ENTITY_KIND_PHONE_PRICE);

				priceEntity.setProperty("source", source);

				priceEntity.setProperty("name", phone.getString("name"));

				priceEntity.setProperty("price", phone.getString("price"));

				priceEntity.setProperty("in_stock", phone.getString("in_stock"));

				priceEntity.setProperty("url", phone.getString("product_url"));

				priceEntities.add(priceEntity);

			} catch (JSONException e) {

				e.printStackTrace();

			}

		}

		ds.put(priceEntities);

	}

	private void initJobQueue()	{

		Entity job1 = createScrapeJob("http://www.flipkart.com/mobiles/all/", "flipkart");

		enqueueScrapeJob(job1);

		Entity job2 = createScrapeJob("http://www.letsbuy.com/mobile-phones-mobiles-c-254_88?perpage=192", "letsbuy");

		enqueueScrapeJob(job2);

		Entity job3 = createScrapeJob("http://www.infibeam.com/Mobiles/search", "infibeam");

		enqueueScrapeJob(job3);

		Entity job4 = createScrapeJob("http://www.homeshop18.com/gsm-handsets/category:3027/", "homeshop18");

		enqueueScrapeJob(job4);

		Entity job5 = createScrapeJob("http://www.flipkart.com/mobiles/tablet-20278", "flipkart");

		enqueueScrapeJob(job5);

		Entity job6 = createScrapeJob("http://www.letsbuy.com/mobile-phones-tablets-c-254_393?perpage=192", "letsbuy");

		enqueueScrapeJob(job6);

		Entity job7 = createScrapeJob("http://www.homeshop18.com/ipads-2f-tablets/category:8937/", "homeshop18");

		enqueueScrapeJob(job7);

//		Entity job8 = createScrapeJob("http://www.adexmart.com/modules/coremanager/modules/filtersearch/filtersearch.json.php?act=filter&ident=16&page=1&perpage=1000&orderby=newest&orderway=desc", "adexmart");

//		enqueueScrapeJob(job8);

	}

	public void doGet(HttpServletRequest req, HttpServletResponse resp)	{

		String cmd = req.getParameter("cmd");

		System.out.println("Command recieved: " + cmd);

		Query q = new Query(ENTITY_KIND_PHONE_PRICE);

		PreparedQuery pq = ds.prepare(q);

		if (cmd != null)	{

			if (cmd.equals("delall"))	{

				System.out.println("Deleting all pricing");

				for (Entity pricing: pq.asIterable())	{

					ds.delete(pricing.getKey());

				}

				System.out.println("Deleting all jobs");

				q = new Query(ENTITY_KIND_JOB);

				pq = ds.prepare(q);

				for (Entity job: pq.asIterable())	{

					ds.delete(job.getKey());

				}

			} else if(cmd.equals("init"))	{

				System.out.println("Initializing the job queue");

				initJobQueue();

			} else if (cmd.equals("getjson")) {

				List<Map<String, String>> entities = new ArrayList<Map<String,String>>();

				Map<String, String> details;

				for (Entity pricing: pq.asIterable())	{

					details = new HashMap<String, String>();

					details.put("source", (String) pricing.getProperty("source"));

					details.put("name", (String) pricing.getProperty("name"));

					details.put("price", (String) pricing.getProperty("price"));

					details.put("in_stock", (String) pricing.getProperty("in_stock"));

					details.put("url", (String) pricing.getProperty("url"));

					entities.add(details);

				}

				resp.setContentType("application/json");

				try {

					resp.getWriter().print(new JSONArray(entities));

				} catch (IOException e) {

					e.printStackTrace();

				}

			}

		}

	}

	public void doPost(HttpServletRequest req, HttpServletResponse resp) {

		String clientId = (String) req.getParameter("id");

		String jobId = (String) req.getParameter("job_id");

		String source = (String) req.getParameter("source");

		String phonePrices = (String) req.getParameter("phone_prices");

		String nextUrl = (String) req.getParameter("next_url");

		Entity job;

		System.out.println(clientId + " " + jobId + " " + source + " " + phonePrices);

		if(clientId.equals("None") || jobId.equals("None") || source.equals("None") || phonePrices.equals("None"))	{

			System.out.println("Assigning first job");

			job = assignJobToScraper(null, null);

		} else	{

			try {

				JSONObject phonePricesJSONObj = new JSONObject("{priceData: " + phonePrices + "}");

//				JSONArray priceJSONArray = new JSONArray(phonePrices);

//				System.out.println(priceJSONArray);

				savePhonePrices(phonePricesJSONObj.getJSONArray("priceData"), source);

//				savePhonePrices(priceJSONArray, source);

				removeJobFromQueue(jobId);	// Removing executed job from the queue

			} catch (JSONException e) {

				e.printStackTrace();

			}

			System.out.println("Assigning next job");

			job = assignJobToScraper(clientId, source);

		}

		if(!nextUrl.equals("None") && !source.equals("None"))	{

			Entity newJob = createScrapeJob(nextUrl, source);

			enqueueScrapeJob(newJob);

		}

		System.out.println("Assigned Job & all attributes set:" + job);

		Map<String, String> scraperJobMap = toMap(job);

		scraperJobMap.put("timetowait", Integer.toString(getTimeToWait()));

		System.out.println("final Map:" + scraperJobMap);

		try {

			resp.getWriter().print(new JSONObject(scraperJobMap));

		} catch (IOException e) {

			e.printStackTrace();

		}

	}

}