Subversion Repositories SmartDukaan

/**

 * 

 */

package in.shop2020.util;

import in.shop2020.metamodel.core.Bullet;

import in.shop2020.metamodel.core.Entity;

import in.shop2020.metamodel.core.Feature;

import in.shop2020.metamodel.core.PrimitiveDataObject;

import in.shop2020.metamodel.core.Slide;

import in.shop2020.metamodel.definitions.BulletDefinition;

import in.shop2020.metamodel.definitions.Catalog;

import in.shop2020.metamodel.definitions.Category;

import in.shop2020.metamodel.definitions.DatatypeDefinition;

import in.shop2020.metamodel.definitions.DefinitionsContainer;

import in.shop2020.metamodel.definitions.FacetDefinition;

import in.shop2020.metamodel.definitions.FacetSlideDefinition;

import in.shop2020.metamodel.definitions.FeatureDefinition;

import in.shop2020.metamodel.util.CreationUtils;

import in.shop2020.metamodel.util.ExpandedBullet;

import in.shop2020.metamodel.util.ExpandedBulletDefinition;

import in.shop2020.metamodel.util.ExpandedEntity;

import in.shop2020.metamodel.util.ExpandedFacetDefinition;

import in.shop2020.metamodel.util.ExpandedFacetSlideDefinition;

import in.shop2020.metamodel.util.ExpandedFeature;

import in.shop2020.metamodel.util.ExpandedFeatureDefinition;

import in.shop2020.metamodel.util.ExpandedSlide;

import java.io.File;

import java.io.FileOutputStream;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import javax.xml.transform.Transformer;

import javax.xml.transform.TransformerFactory;

import javax.xml.transform.stream.StreamResult;

import javax.xml.transform.stream.StreamSource;

import org.apache.commons.lang.StringEscapeUtils;

import org.apache.commons.lang.StringUtils;

/**

 * Command line utility to convert IR Definitions into IR Data and IR Meta-data

 * 

 * @author rajveer

 *

 */

public class NewIR {

	/**

	 * Level - 4, 8, 12, 16

	 */

	private String[] xmlIndentation = {"", "    ", "        ", "            ","                "};

	private String[] xmlTabIndentation = {"", "\t", "\t\t", "\t\t\t", "\t\t\t\t", "\t\t\t\t\t", "\t\t\t\t\t\t"};

	private Map<Long, List<String>> facetIDFacetValues = new HashMap<Long, List<String>>();

	private static Map<String, Map<String, String>> synonymMap = null;

	static {

		try {

			synonymMap = CreationUtils.getSynonyms();

		}catch(Exception e){

			e.printStackTrace();

		}

	}

	List<Entity> entities;

	/**

	 * @param args

	 * @throws Exception 

	 */

	public static void main(String[] args) throws Exception {

		/*

		NewIR ir = new NewIR();

		ir.exportIRData();

		ir.transformIrDataXMLtoSolrXML();

		ir.exportIRMetaData();

		ir.transformIrMetaDataXMLtoSolrSchemaXML();

		ir.transformIrMetaDataXMLtoSolrCatchAllXML();

		*/

	}

	/**

	 * 

	 * @param entityIdItemMap

	 */

	public NewIR(List<Entity> entities){

	    this.entities = entities;

	}

	/**

	 * 

	 * @param inXMLFilename

	 * @param xslFilename

	 * @param outXMLFilename

	 * @throws Exception 

	 */

	public void xsltTransformation(String inXMLFilename, String xslFilename, String outXMLFilename) throws Exception{

		// Use the static TransformerFactory.newInstance() method to instantiate 

		  // a TransformerFactory. The javax.xml.transform.TransformerFactory 

		  // system property setting determines the actual class to instantiate --

		  // org.apache.xalan.transformer.TransformerImpl.

		TransformerFactory tFactory = TransformerFactory.newInstance();

			// Use the TransformerFactory to instantiate a Transformer that will work with  

			// the stylesheet you specify. This method call also processes the stylesheet

		  // into a compiled Templates object.

		Transformer transformer = tFactory.newTransformer(new StreamSource(xslFilename));

			// Use the Transformer to apply the associated Templates object to an XML document

			// (foo.xml) and write the output to a file (foo.out).

		transformer.transform(new StreamSource(inXMLFilename), new StreamResult(new FileOutputStream(outXMLFilename)));

	}

	/**

	 * 

	 * @throws Exception

	 */

	public void transformIrDataXMLtoSolrXML(long catalogId) throws Exception {

		String irDataFilename = Utils.EXPORT_PATH + "xml/intermediate/" + catalogId +  "_irdata.xml";

		String irSolrDataFilename = Utils.EXPORT_PATH + "xml/final/" +  catalogId +  "_irdata_solr.xml";

		String irXslFilename = "src/xsl/irdata_solrdata.xsl";

		System.out.println(irSolrDataFilename);

		File solrFile = new File(irSolrDataFilename);

		if(!solrFile.exists()){

			solrFile.createNewFile();

		}

		xsltTransformation(irDataFilename, irXslFilename, irSolrDataFilename);

	}

	/**

	 * 

	 * @throws Exception

	 */

	public void transformIrMetaDataXMLtoSolrSchemaXML() throws Exception {

		String irDataFilename = Utils.EXPORT_PATH  + "xml/intermediate/" + "irmetadata.xml";

		String irSolrDataFilename = Utils.EXPORT_PATH + "xml/final/" + "irmetadata_solrschema.xml";

		String irXslFilename = "src/xsl/irmetadata_solrschema.xsl";

		System.out.println(irSolrDataFilename);

		File solrFile = new File(irSolrDataFilename);

		if(!solrFile.exists()){

			solrFile.createNewFile();

		}

		xsltTransformation(irDataFilename, irXslFilename, irSolrDataFilename);

	}

	/**

	 * 

	 * @throws Exception

	 */

	public void transformIrMetaDataXMLtoSolrCatchAllXML() throws Exception {

		String irDataFilename = Utils.EXPORT_PATH  + "xml/intermediate/" + "irmetadata.xml";

		String irSolrDataFilename = Utils.EXPORT_PATH + "xml/final/" + "irmetadata_catchall.xml";

		String irXslFilename = "src/xsl/irmetadata_catchall.xsl";

		System.out.println(irSolrDataFilename);

		File solrFile = new File(irSolrDataFilename);

		if(!solrFile.exists()){

			solrFile.createNewFile();

		}

		xsltTransformation(irDataFilename, irXslFilename, irSolrDataFilename);

	}

	/**

	 * 

	 * @throws Exception

	 */

	public void exportIRMetaData() throws Exception {

		DefinitionsContainer defs = 

			Catalog.getInstance().getDefinitionsContainer();

		// <IRMetaData>

		List<String> xmlSnippets = new ArrayList<String>();

		xmlSnippets.add("<IRMetaData>");

		xmlSnippets.add("\t<Facets>");

		IRMetaDataJythonWrapper jy = new IRMetaDataJythonWrapper();

		// Iterate over all facet definitions

		Map<Long, FacetDefinition> facetDefs = defs.getFacetDefinitions();

		for(FacetDefinition facetDef : facetDefs.values()) {

			jy.reset();

			jy.initialize();

			ExpandedFacetDefinition expFacetDef = 

				new ExpandedFacetDefinition(facetDef);

			jy.setExpandedFacetDefinition(expFacetDef);

			jy.executeRule();

			String facetXMLSnip = jy.getXMLSnippet();

			Utils.info("facetXMLSnip=" + facetXMLSnip);

			xmlSnippets.add(facetXMLSnip);

		}

		xmlSnippets.add("\t</Facets>");

		xmlSnippets.add("\n\t<Properties>");

		// Iterate over all feature definitions

		Map<Long, FeatureDefinition> featureDefs = defs.getFeatureDefinitions();

		for(FeatureDefinition featureDef : featureDefs.values()) {

			String propertyXMLSnip = this.getPropertyXMLSnippet(featureDef);

			Utils.info("propertyXMLSnip=" + propertyXMLSnip);

			xmlSnippets.add(propertyXMLSnip);

		}

		xmlSnippets.add("\t</Properties>");

		xmlSnippets.add("\n\t<Categories>");

		Category rootCategory = defs.getCategory(Catalog.getInstance().getRootCategory().getID());

		String categoryXMLSnip = this.getCategoryXMLSnippet(rootCategory, 2);

		Utils.info("categoryXMLSnip=" + categoryXMLSnip);

		xmlSnippets.add("\t</Categories>");

		// </IRMetaData>

		xmlSnippets.add("</IRMetaData>");

		String irMetaDataXML = StringUtils.join(xmlSnippets, "\n");

		Utils.info(irMetaDataXML);

		// Write it to file

		String irMetaDataFilename = Utils.EXPORT_PATH +  "xml/intermediate/" + "irmetadata.xml";

		DBUtils.store(irMetaDataXML, irMetaDataFilename);

	}

	/**

	 * 

	 * @param category

	 * @return

	 * @throws Exception 

	 */

	private String getCategoryXMLSnippet(Category category, int indent) 

		throws Exception {

		DefinitionsContainer defs = 

			Catalog.getInstance().getDefinitionsContainer();

		String xmlSnip = this.xmlTabIndentation[indent] + "<Category";

		xmlSnip += " ID=\"" + category.getID() + "\"";

		xmlSnip += " label=\"" + category.getLabel() + "\"";

		xmlSnip += ">\n";

		List<Category> children = category.getChildrenCategory();

		if(children != null) {

			for(Category child : children) {

				xmlSnip += this.getCategoryXMLSnippet(child, indent+1);

			}

		}

		else {

			// Only leaf category will have entities

			// Facet IDs

			xmlSnip += this.xmlTabIndentation[indent+1] + "<Facets>\n";

			List<Long> facetDefIDs = 

				defs.getFacetDefinitionIDs(category.getID());

			//Utils.info("facetDefIDs=" + facetDefIDs);

			if( facetDefIDs != null && !facetDefIDs.isEmpty() ){

					for(Long facetDefID : facetDefIDs) {

					xmlSnip += this.xmlTabIndentation[indent+2] + 

						"<FacetID>" + facetDefID + "</FacetID>\n";

				}

			}

			xmlSnip += this.xmlTabIndentation[indent+1] + "</Facets>\n\n";

			// Feature IDs

			xmlSnip += this.xmlTabIndentation[indent+1] + "<Properties>\n";

			List<Long> featureDefIDs = 

				defs.getFeatureDefinitionIDs(category.getID());

			//Utils.info("featureDefIDs=" + featureDefIDs);

			for(Long featureDefID : featureDefIDs) {

				xmlSnip += this.xmlTabIndentation[indent+2] + 

					"<FeatureID>" + featureDefID + "</FeatureID>\n";

			}

			xmlSnip += this.xmlTabIndentation[indent+1] + "</Properties>\n";

		}

		xmlSnip += this.xmlTabIndentation[indent] + "</Category>\n";

		return xmlSnip;

	}

	/**

	 * 

	 * @param featureDef

	 * @return

	 * @throws Exception 

	 */

	private String getPropertyXMLSnippet(FeatureDefinition featureDef) 

		throws Exception {

		String xmlSnip = "\t\t<Property";

		xmlSnip += " ID=\"" + featureDef.getID() + "\"";

		xmlSnip += " label=\"" + featureDef.getLabel() + "\"";

		ExpandedFeatureDefinition expFeatureDef = 

			new ExpandedFeatureDefinition(featureDef);

		ExpandedBulletDefinition expBulletDef = 

			expFeatureDef.getExpandedBulletDefinition();

		String datatype = "string";

		if(expBulletDef.isComposite() || expBulletDef.isEnumerated()) {

			datatype = "string";

		}

		else {

			DatatypeDefinition datatypeDef = 

				expBulletDef.getDatatypeDefinition();

			datatype = datatypeDef.getName();

			// REVISIT

			if(datatype.equals("hours_mins") || datatype.equals("days_hours") ||

					datatype.equals("hours_mins") || 

					datatype.equals("days_hours")) {

				datatype = "string";

			}

		}

		xmlSnip += " datatype=\"" + datatype + "\"";

		String multivalue = "false";

		if (expBulletDef.isMultivalue()) {

			multivalue = "true";

		}

		xmlSnip += " isMultivalue=\"" + multivalue + "\"";

		xmlSnip += "/>";

		return xmlSnip;

	}

	private String processFacet(FacetDefinition facet, ExpandedSlide expSlide, ExpandedFacetSlideDefinition expFacetSlideDef, Long facetDefID) throws Exception {

		IRDataJythonWrapper jw = new IRDataJythonWrapper();

		jw.setExpandedSlide(expSlide);

		jw.setExpandedFacetSlideDefinition(expFacetSlideDef);

		// Execute Python script

		jw.executeRule();

		//FIXME

		// Normalize

//		if(defs.needsNormalization(feature.getFeatureDefinitionID())) {

//			Utils.info("needsNormalization feature=" + feature.getFeatureDefinitionID());

//			

//			feature = this.normalize(feature);

//		}

		List<Object> values = (List<Object>)jw.getValues();

		Utils.info("values=" + values);

		// Append to facet values

		Utils.info("facetDefID=" + facetDefID);

		List<String> facetValues = this.facetIDFacetValues.get(

				new Long(facetDefID));

		if(facetValues == null) {

			facetValues = new ArrayList<String>();

			this.facetIDFacetValues.put(new Long(facetDefID), facetValues);

		}

		if(values != null) {

			for(Object value : values) {

				String strValue = value.toString();

				if(!facetValues.contains(strValue)) {

					facetValues.add(strValue);

				}

			}

		}

		ExpandedFacetDefinition expFacet = new ExpandedFacetDefinition(facet);

		// Parse returned Python list

		String facetXMLSnip = null;

		if(values != null) {

			facetXMLSnip = this.getFacetXMLSnippet(expFacet, values, 2);

		}

		Utils.info("0 facetXMLSnip=" + facetXMLSnip);

		return facetXMLSnip;

	}

	/**

	 * 

	 * @param expFacetDef

	 * @param values

	 * @param indent

	 * @return String XML Snippet

	 */

	private String getFacetXMLSnippet(ExpandedFacetDefinition expFacetDef,	List<Object> values, int indent) {

		// <Facet Label="Form Factor">

		List<String> xmlSnippet = new ArrayList<String>();

		String target = expFacetDef.getTarget();

		xmlSnippet.add(this.xmlIndentation[indent] + "<Facet ID=\"" + expFacetDef.getID() + "\" Label=\"" + target + "\">");

		//<Value>Candybar</Value>

		for(Object value : values) {

			xmlSnippet.add(this.xmlIndentation[indent + 1] + "<Value>" + StringEscapeUtils.escapeXml(value.toString()) + "</Value>");

		}

		//</Facet>

		xmlSnippet.add(this.xmlIndentation[indent] + "</Facet>");

		return StringUtils.join(xmlSnippet, "\n");

	}

	/**

	 * @throws Exception 

	 * 

	 */

	public void exportIRData() throws Exception {

		DefinitionsContainer defs = Catalog.getInstance().getDefinitionsContainer();

		Map<Long, List<FacetDefinition>> slideFacets = defs.getSlideFacetDefinitions();

		for(Entity entity: entities){

			try{

			Utils.info(entity.getID());

			// <IRData>

			List<String> irDataXMLSnippets = new ArrayList<String>();

			irDataXMLSnippets.add("<IRData>");

		    long entityID = entity.getID();

		    //Setting minPrice zero because irdatarule requires float. It wont accept string 

		    double minPrice = 0;

			ExpandedEntity expEntity = new ExpandedEntity(entity);

			long categoryID = expEntity.getCategoryID();

			List<ExpandedSlide> expSlides = expEntity.getExpandedSlides();

			List<String> entityXMLSnippets = new ArrayList<String>();

			entityXMLSnippets.add(this.xmlIndentation[1] + "<Entity ID=\""+ entityID +"\">");

			String parentCategory = expEntity.getCategory().getLabel();

			entityXMLSnippets.add(this.xmlIndentation[2] + "<Category>" + StringEscapeUtils.escapeXml(parentCategory) + "</Category>");

			String title = StringEscapeUtils.escapeXml(expEntity.getBrand()) + " " + StringEscapeUtils.escapeXml(expEntity.getModelName()) + 

				((expEntity.getModelNumber() != null) ? (" " +	StringEscapeUtils.escapeXml(expEntity.getModelNumber())): "");

			entityXMLSnippets.add(this.xmlIndentation[2] +	"<Title>" + title + "</Title>");

			//Boost titles for the mobile phones, laptops and tablets only

			if(expEntity.getCategory().getParentCategory().isHasAccessories() || expEntity.getCategory().getParentCategory().getID() == Utils.TABLETS_CATEGORY || expEntity.getCategory().getParentCategory().getID() == Utils.LAPTOPS_CATEGORY){

			    entityXMLSnippets.add(this.xmlIndentation[2] +   "<Boost>" + 5 + "</Boost>");

			}else{

			    entityXMLSnippets.add(this.xmlIndentation[2] +   "<Boost>" + 0 + "</Boost>");

			}

			String subCategory = defs.getCategory(categoryID).getLabel();

			String  mainCategory = defs.getCategory(categoryID).getParentCategory().getLabel();

			long mainCatId = defs.getCategory(categoryID).getParentCategory().getID();

			if(mainCatId == Utils.ROOT_CATAGOEY){

				mainCategory = subCategory;

			}

			String brand = expEntity.getBrand();

			//Create zero slide

			Slide zeroSlide = createZeroSlide(brand, minPrice, mainCategory, subCategory, "In Stock");

			ExpandedSlide expandedZeroSlide = new ExpandedSlide(zeroSlide);

			expSlides.add(expandedZeroSlide);

			for(ExpandedSlide expSlide: expSlides){

				System.out.println(expSlide.getSlideDefinitionID());

				if(slideFacets.containsKey(expSlide.getSlideDefinitionID())){

					List<FacetDefinition> facets = slideFacets.get(expSlide.getSlideDefinitionID());

					for(FacetDefinition facet: facets){

						for(FacetSlideDefinition facetSlides: facet.getFacetSlideDefinitions()){

							if(facetSlides.getSlideDefinitionID() != expSlide.getSlideDefinitionID()){continue;}

							ExpandedFacetSlideDefinition expFacetSlideDef = new ExpandedFacetSlideDefinition(facetSlides) ;

							for(ExpandedFeature expFeature: expSlide.getExpandedFeatures()){

								if(defs.needsNormalization(expFeature.getFeatureDefinitionID())) {

									Utils.info("needsNormalization feature=" + expFeature.getFeatureDefinitionID());

									//expFeature = this.normalize(expFeature);

								}

							}

							entityXMLSnippets.add(processFacet(facet, expSlide, expFacetSlideDef, facet.getID()));

						}

					}

				}

			}

			// Collect PROPERTIES

			List<Long>facetFeatureIds = new ArrayList<Long>();

			facetFeatureIds.add(Utils.BRAND_FEATURE_DEFINITION_ID);

			facetFeatureIds.add(Utils.MAIN_CAT_FEATURE_DEFINITION_ID);

			facetFeatureIds.add(Utils.SUB_CAT_FEATURE_DEFINITION_ID);

			facetFeatureIds.add(Utils.PRICE_FEATURE_DEFINITION_ID);

			facetFeatureIds.add(Utils.TAG_FEATURE_DEFINITION_ID);

			String propertiesXMLSnippetsStr = this.getPropertiesXMLSnippet(expEntity, facetFeatureIds, 2);

			entityXMLSnippets.add(propertiesXMLSnippetsStr);

			entityXMLSnippets.add(this.xmlIndentation[1] + "</Entity>");

			System.out.println(entityXMLSnippets);

			irDataXMLSnippets.addAll(entityXMLSnippets);

			irDataXMLSnippets.add("</IRData>");

			String irDataXML = StringUtils.join(irDataXMLSnippets, "\n");

			Utils.info(irDataXML);

			// Write it to file

			String irDataFilename = Utils.EXPORT_PATH + "xml/intermediate/" + entityID + "_irdata.xml";

			DBUtils.store(irDataXML, irDataFilename);

			transformIrDataXMLtoSolrXML(entityID);

		} catch (Exception e){

			Utils.info("Could not process entity:" + entity.getID());

			//e.printStackTrace();

		}

		}

	}

	private Slide createZeroSlide(String brand, double price, String mainCategory, String subCategory, String availability) {

		Slide zeroSlide = new Slide(Utils.ZERO_SLIDE_DEFINITION_ID);

		List<Feature> zeroSlideFeatures = new ArrayList<Feature>();

		Feature brandFeature = new Feature(Utils.BRAND_FEATURE_DEFINITION_ID);

		Bullet brandBullet = new Bullet(new PrimitiveDataObject(brand));

		List<Bullet> brandBullets = new ArrayList<Bullet>();

		brandBullets.add(brandBullet);

		brandFeature.setBullets(brandBullets);

		zeroSlideFeatures.add(brandFeature);

		Feature availabilityFeature = new Feature(Utils.AVAILABILITY_FEATURE_DEFINITION_ID);

		Bullet availabilityBullet = new Bullet(new PrimitiveDataObject(availability));

		List<Bullet> availabilityBullets = new ArrayList<Bullet>();

		availabilityBullets.add(availabilityBullet);

		availabilityFeature.setBullets(availabilityBullets);

		zeroSlideFeatures.add(availabilityFeature);

		Feature priceFeature = new Feature(120128);

		Bullet priceBullet = new Bullet(new PrimitiveDataObject((new Double(price)).toString()));

		List<Bullet> priceBullets = new ArrayList<Bullet>();

		priceBullets.add(priceBullet);

		priceFeature.setBullets(priceBullets);

		zeroSlideFeatures.add(priceFeature);

		Feature mainCategoryFeature = new Feature(120123);

		Bullet mainCategoryBullet = new Bullet(new PrimitiveDataObject(mainCategory));

		List<Bullet> mainCategoryBullets = new ArrayList<Bullet>();

		mainCategoryBullets.add(mainCategoryBullet);

		mainCategoryFeature.setBullets(mainCategoryBullets);

		zeroSlideFeatures.add(mainCategoryFeature);

		Feature subCategoryFeature = new Feature(120124);

		Bullet subCategoryBullet = new Bullet(new PrimitiveDataObject(subCategory));

		List<Bullet> subCategoryBullets = new ArrayList<Bullet>();

		subCategoryBullets.add(subCategoryBullet);

		subCategoryFeature.setBullets(subCategoryBullets);

		zeroSlideFeatures.add(subCategoryFeature);

		try {

			String brandSynonyms = synonymMap.get("brand").get(brand);

			String subCategorySynonyms = synonymMap.get("subcategory").get(subCategory);

			if(brandSynonyms != null){

				Feature brandSynonymFeature = new Feature(Utils.BRAND_SYNONYMS_FEATURE_DEFINITION_ID);

				List<Bullet> brandSynonymBullets = new ArrayList<Bullet>();

				for(String brSyn : brandSynonyms.split(",")){

					Bullet brandSynonymBullet = new Bullet(new PrimitiveDataObject(brSyn));

					brandSynonymBullets.add(brandSynonymBullet);

				}

				brandSynonymFeature.setBullets(brandSynonymBullets);

				zeroSlideFeatures.add(brandSynonymFeature);

			}

			if(subCategorySynonyms != null){

				Feature subCategorySynonymFeature = new Feature(Utils.SUB_CATEGORY_SYNONYMS_FEATURE_DEFINITION_ID);

				List<Bullet> subCategorySynonymBullets = new ArrayList<Bullet>();

				for(String subCatSyn : subCategorySynonyms.split(",")){

					Bullet subCategorySynonymBullet = new Bullet(new PrimitiveDataObject(subCatSyn));

					subCategorySynonymBullets.add(subCategorySynonymBullet);

				}

				subCategorySynonymFeature.setBullets(subCategorySynonymBullets);

				zeroSlideFeatures.add(subCategorySynonymFeature);

			}

		} catch (Exception e) {

			e.printStackTrace();

		}

		zeroSlide.setFeatures(zeroSlideFeatures);

		return zeroSlide;

	}

	/**

	 * 

	 * @param feature

	 * @return Feature

	 * @throws Exception 

	 */

	private ExpandedFeature normalize(ExpandedFeature expFeature) throws Exception {

		BulletDefinition bulletDef = 

			expFeature.getFeatureDefinition().getBulletDefinition();

		ExpandedBulletDefinition expBulletDef = new ExpandedBulletDefinition(

				bulletDef);

		if(expBulletDef.isPrimitive()) {

			PrimitiveNormalizationJythonWrapper jy = 

				new PrimitiveNormalizationJythonWrapper();

			jy.setExpandedFeature(expFeature);

			jy.excuteRule();

			String newValue = jy.getNewValue();

			long newUnitID = jy.getNewUnitID();

			List<Bullet> newBullets = new ArrayList<Bullet>();

			Bullet newBullet = new Bullet(new PrimitiveDataObject(newValue));

			newBullet.setUnitID(newUnitID);

			newBullets.add(newBullet);

			expFeature.setBullets(newBullets);

		}

		else {

			Utils.severe("Normalization not defined for non-primitives");

		}

		return expFeature;

	}

	/**

	 * 

	 * @param expEntity

	 * @param facetFeatureIDs

	 * @param indent

	 * @return

	 */

	private String getPropertiesXMLSnippet(ExpandedEntity expEntity, 

			List<Long> facetFeatureIDs, int indent) {

		List<String> xmlSnippet = new ArrayList<String>();

		// Collect all free-form content here

		List<String> ffc = new ArrayList<String>();

		// Features

		List<ExpandedSlide> expSlides = expEntity.getExpandedSlides();

		for(ExpandedSlide expSlide : expSlides) {

			List<ExpandedFeature> expFeatures = expSlide.getExpandedFeatures();

			if(expSlide.getFreeformContent() != null) {

				ffc.add(expSlide.getFreeformContent().getFreeformText());

			}

			if(expFeatures == null) {

				continue;

			}

			for(ExpandedFeature expFeature : expFeatures) {

				Long featureDefID = 

					new Long(expFeature.getFeatureDefinitionID());

				// FFC at feature level

				if(expFeature.getFreeformContent() != null) {

					ffc.add(expFeature.getFreeformContent().getFreeformText());

				}

				// Exclude those who are already covered as facets

				// and the ones that are marked false for indexing

				if(facetFeatureIDs.contains(featureDefID) || !expFeature.getFeatureDefinition().isIndexed()) {

					continue;

				}

				List<ExpandedBullet> expBullets = 

					expFeature.getExpandedBullets();

				if(expBullets == null) {

					continue;

				}

				//<Property Label="">

				xmlSnippet.add(this.xmlIndentation[indent] + 

						"<Property ID=\"" + expFeature.getFeatureDefinitionID() 

						+ "\" Label=\"" + StringEscapeUtils.escapeXml(

								expFeature.getFeatureDefinition().getLabel()) + 

						"\">");

				//<Value></Value>

				for(ExpandedBullet bullet : expBullets) {

					if(bullet.getValue().trim().equalsIgnoreCase("")){

						continue;

					}

					// FFC at bullet level

					if(bullet.getFreeformContent() != null) {

						ffc.add(bullet.getFreeformContent().getFreeformText());

					}

					xmlSnippet.add(this.xmlIndentation[indent + 1] + "<Value>" + 

							StringEscapeUtils.escapeXml(bullet.getValue()) + 

							"</Value>");

				}

				//</Property>

				xmlSnippet.add(this.xmlIndentation[indent] + "</Property>");

			}

		}

		// FFC as Label="Free-form Content"

		if(!ffc.isEmpty()) {

			xmlSnippet.add(this.xmlIndentation[indent] + 

					"<Property ID=\"000000\" Label=\"Free-form Content\">");

			for(String f : ffc) {

				if(f != null) {

					f = StringEscapeUtils.escapeXml(f);

					xmlSnippet.add(this.xmlIndentation[indent + 1] + "<Value>" + 

						f + "</Value>");

				}

			}

			xmlSnippet.add(this.xmlIndentation[indent] + "</Property>");

		}

		// Children slides

		// TODO

		return StringUtils.join(xmlSnippet, "\n");

	}

}