Subversion Repositories SmartDukaan

/**
 * 
 */
package in.shop2020.metamodel.util;

import in.shop2020.metamodel.core.Bullet;
import in.shop2020.metamodel.core.CompositeDataObject;
import in.shop2020.metamodel.core.Entity;
import in.shop2020.metamodel.core.EnumDataObject;
import in.shop2020.metamodel.core.Feature;
import in.shop2020.metamodel.core.FreeformContent;
import in.shop2020.metamodel.core.PrimitiveDataObject;
import in.shop2020.metamodel.core.Slide;
import in.shop2020.metamodel.definitions.BulletDefinition;
import in.shop2020.metamodel.definitions.Catalog;
import in.shop2020.metamodel.definitions.Category;
import in.shop2020.metamodel.definitions.CompositeDefinition;
import in.shop2020.metamodel.definitions.CompositePartDefinition;
import in.shop2020.metamodel.definitions.DatatypeDefinition;
import in.shop2020.metamodel.definitions.DefinitionsContainer;
import in.shop2020.metamodel.definitions.EditorialImportance;
import in.shop2020.metamodel.definitions.EntityContainer;
import in.shop2020.metamodel.definitions.EnumDefinition;
import in.shop2020.metamodel.definitions.FeatureDefinition;
import in.shop2020.metamodel.definitions.SlideDefinition;
import in.shop2020.metamodel.definitions.SlideFeatureDefinition;
import in.shop2020.metamodel.definitions.Unit;
import in.shop2020.util.DBUtils;
import in.shop2020.util.Utils;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.LineNumberReader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;

import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;

/**
 * NO LONGER USED - POI BASED DATA EXTRACTION FROM MS PPT FILE HAS REPLACED IT
 * 
 * CN - Content Tool
 * Imports content from text and exports content. Validates against content model
 * 
 * @author naveen
 *
 */
public class CN {
        public static final String CONTENT_SRC_TXT_PATH = 
                "/home/naveen/workspace/eclipse/content/txt/";
        public static final String CONTENT_SRC_HTML_PATH = 
                "/home/naveen/workspace/eclipse/content/html/";
        public static final String CONTENT_SRC_XML_PATH = 
                "/home/naveen/workspace/eclipse/content/xml/";
        
        //** LOCAL
        public static final String CONTENT_DB_PATH =  
                "/home/naveen/workspace/eclipse/db/entities/";
        
        /** WEB
        public static final String CONTENT_DB_PATH =  
                "/var/lib/tomcat6/webapps/shop2020/db/entities/";
        */
        
        
        // Known patterns
        public static final String PATTERN_LEVEL_1 = "* ";
        public static final String PATTERN_LEVEL_2 = "   * ";
        public static final String PATTERN_LEVEL_3 = "      * ";
        public static final String PATTERN_LEVEL_4 = "         * ";

        private long categoryID;
        private String srcFile;
        private String dbFile;
        
        private String introductionTitle;
        
        // Slide names in sequence
        private List<String> slideNames = new ArrayList<String>();
        
        private Map<String, Collection<String>> slideNameToFeatureLabels = 
                new HashMap<String, Collection<String>>();
        
        private Map<String, Long> slideNameToSlideDefID = 
                new HashMap<String, Long>();
        
        private Map<String, Long> slideNameFeatureNameToFeatureDefID =
                new HashMap<String, Long>();

        private Map<String, List<Bullet>> slideNameFeatureNameToBullets =
                new HashMap<String, List<Bullet>>();
        
        private Map<String, String> slideNameToFreeformContent = 
                new HashMap<String, String>();
                
        /**
         * @param args
         */
        public static void main(String[] args) throws Exception {
                String[] commands = new String[] {"show", "import"};
                
                String usage = "Usage: CN ["+ StringUtils.join(commands, "|") +
                        "] [{Entity ID}|{Category ID} {Content file name}]\n";
                
                if(args.length < 2) {
                        System.out.println(usage);
                        System.exit(-1);
                }
                
                String inputCommand = args[0];
                
                if(!ArrayUtils.contains(commands, inputCommand)) {
                        System.out.println(usage);
                        System.exit(-1);
                }

                if (inputCommand.equals("show")) {
                        String entityID = args[1];
                        CN cn = new CN();
                        cn.showEntity(new Long(entityID).longValue());
                        System.exit(0);
                }
                
                if (inputCommand.equals("import")) {
                        String inputCategoryID = args[1];
                        if(args.length < 3) {
                                System.out.println(usage);
                                System.exit(-1);
                        }

                        String inputFilename = args[2];
                        long categoryID = new Long(inputCategoryID).longValue();
                        CN cn = new CN(categoryID, inputFilename);
                        cn.importEntity();
                        System.exit(0);
                }
        }
        
        /**
         * 
         */
        public CN() {
        }
        
        /**
         * 
         * @param categoryID
         * @param fileName
         */
        public CN(long categoryID, String fileName) {
                this.categoryID = categoryID;
                
                this.srcFile = CONTENT_SRC_TXT_PATH + fileName + ".txt";
                this.dbFile = CONTENT_DB_PATH + "entities" + ".ser";
        }
        
        /**
         * 
         * @throws Exception
         */
        public void showEntity(long entityID) throws Exception {
                EntityContainer entContainer = 
                        Catalog.getInstance().getEntityContainer();
                
                ExpandedEntity expandedEntity = 
                        entContainer.getExpandedEntity(entityID);
                
                Utils.logger.info("expandedEntity=" + expandedEntity);
                
                /*
                Entity entity = entContainer.getEntity(entityID);
                Utils.logger.info("Entity=" + entity);
                */
                List<ExpandedSlide> expslides = expandedEntity.getExpandedSlides();

                for(ExpandedSlide expslide : expslides) {
                        String slidelabel = expslide.getSlideDefinition().getLabel();
                        System.out.println(slidelabel);
                        
                        List<ExpandedFeature> expfeatures = expslide.getExpandedFeatures();
                        
                        for(ExpandedFeature expfeature : expfeatures ) {
                                String featurelabel = expfeature.getFeatureDefinition().getLabel();
                                System.out.println("\t"+featurelabel);
                                
                                List<ExpandedBullet> expbullets = 
                                                expfeature.getExpandedBullets();
                                
                                if(expbullets == null) continue;
                                
                                for (ExpandedBullet expbullet : expbullets) {
                                        System.out.println("\t\t" + expbullet.displayText());
                                }
                        }
                }
        }
        
        /**
         * 
         * @throws Exception
         */
        public void importEntity() throws Exception {           
                //this.test();
                Utils.logger.info(this.srcFile);
                
                FileReader fileReader = new FileReader(this.srcFile);
                BufferedReader lineReader = new BufferedReader(fileReader);
                LineNumberReader reader = new LineNumberReader(lineReader);
                
                List<String> allLines = new ArrayList<String>();
                
                // 1 Collect all candidate slide names
                // Order of keys need to be restored
                Map<Integer, String> candidateSlideNames = 
                        new TreeMap<Integer, String>();
                
                int lineCount = 0;
                boolean first = true;
                while(true) {
                        String line = reader.readLine();                        
                        if(line == null) {
                                break;
                        }
                        
                        allLines.add(line);
                        lineCount++;
                        
                        if(line.isEmpty()) {
                                continue;
                        }
                        

                        int lineNumber = reader.getLineNumber();
                        
                        Utils.logger.info(line);
                        //Utils.logger.info("lineNumber="+lineNumber);

                        // First non-empty line is introduction title - brand, model
                        if(first) {
                                this.introductionTitle = StringUtils.strip(line);
                                first = false;
                        }
                        
                        // RE-VISIT
                        // 1.1 Slide names starting with Alpha
                        //Utils.logger.info(line.substring(0, 1));
                        if(StringUtils.isAlpha((line.substring(0, 1)))) {
                                //Utils.logger.info(line);
                                //Utils.logger.info("Level 0");
                                candidateSlideNames.put(new Integer(lineNumber), 
                                                StringUtils.strip(line));
                        }
                }
                Utils.logger.info("introductionTitle=" + this.introductionTitle);
                //System.exit(0);

                lineReader.close();
                
                Utils.logger.info("lineCount=" + lineCount);
                Utils.logger.info(candidateSlideNames.toString());
                
                // 2 Filter and validate parent slide names
                Map<Integer, String> slideNames = 
                        this.processSlides(candidateSlideNames);
                
                Utils.logger.info(slideNames.toString());
                
                // 3 Collect all children slides
                // 4 Filter and validate children slide names
                // TODO
                
                // 5 Collect all Features for a slide
                // 5.1 Find line number range between which content for feature is found
                Map<String, int[]> featureLineNumberRangePerSlide = 
                        new HashMap<String, int[]>();
                        
                Utils.logger.info("slideNames.keySet=" + 
                                slideNames.keySet().toString());
                
                Integer[] slideLineNumbers = slideNames.keySet().toArray(
                                new Integer[0]);
                
                for(int i=0; i<slideLineNumbers.length-1; i++) {
                        int from = slideLineNumbers[i]+1;
                        int to = slideLineNumbers[i+1]-1;
                        Utils.logger.info("from=" + from + " to=" + to);
                        
                        String tSlideName = slideNames.get(
                                        new Integer(slideLineNumbers[i]));
                        
                        Utils.logger.info("tSlideName=" + tSlideName);
                        
                        featureLineNumberRangePerSlide.put(tSlideName, 
                                        new int[] {from, to});
                }

                // For last slide
                Utils.logger.info("tSlideName=" + 
                                slideNames.get(new Integer(
                                                slideLineNumbers[slideLineNumbers.length-1])));
                
                Utils.logger.info("lineNumbers[lineNumbers.length-1]=" + 
                                slideLineNumbers[slideLineNumbers.length-1]);
                
                Utils.logger.info(
                        "from=" + (slideLineNumbers[slideLineNumbers.length-1]+1) + " to=" +
                        (lineCount-1));
                
                featureLineNumberRangePerSlide.put(
                        slideNames.get(
                                new Integer(slideLineNumbers[slideLineNumbers.length-1])), 
                                new int[] {(slideLineNumbers[slideLineNumbers.length-1] + 1), 
                                (lineCount-1)});
                
                Utils.logger.info("featureLineNumberRangePerSlide=" + 
                                featureLineNumberRangePerSlide.toString());
                
                // 6 Filter and validate features
                DefinitionsContainer defs = 
                        Catalog.getInstance().getDefinitionsContainer();
                
                Map<String, Map<Integer, String>> slideFeatureLabels = 
                        new HashMap<String, Map<Integer, String>>();
                
                for(String slideName : slideNames.values()) {
                        //Utils.logger.info("slideName=" + slideName);
                        List<SlideDefinition> slideDefs = 
                                defs.getSlideDefinitions(this.categoryID, slideName);
                        
                        if(slideDefs == null) {
                                continue;
                        }
                        
                        if (slideDefs.isEmpty()) {
                                continue;
                        } 
                        
                        // RE-VISIT
                        // Pick the first
                        SlideDefinition slideDef = slideDefs.get(0);
                        Utils.logger.info("slideDef=" + slideDef);
                        
                        Long slideID = slideDef.getID();
                        this.slideNameToSlideDefID.put(slideName, slideID);
                        
                        // If there are NO children slides or features defined for the slide
                        // Every under it is free-form content
                        List<Long> childrenSlideDefIDs = 
                                slideDef.getChildrenSlideDefinitionIDs();
                        
                        List<SlideFeatureDefinition> slideFeatureDef = 
                                slideDef.getSlideFeatureDefinitions();
                        
                        if((childrenSlideDefIDs == null || childrenSlideDefIDs.size() == 0) 
                                && (slideFeatureDef == null || slideFeatureDef.size() == 0)){
                                
                                int[] slideFFCRange = 
                                        featureLineNumberRangePerSlide.get(slideName);
                                
                                StringBuffer ffcSB = new StringBuffer();
                                Utils.logger.info("slideFFCRange[0]=" + slideFFCRange[0]);
                                Utils.logger.info("slideFFCRange[1]=" + slideFFCRange[1]);
                                
                                for(int i=slideFFCRange[0]; i<slideFFCRange[1]; i++) {
                                        ffcSB.append(allLines.get(i));
                                }
                                
                                Utils.logger.info(slideName + " " + " ffcSB.toString()=" + 
                                                ffcSB.toString());
                                
                                this.slideNameToFreeformContent.put(slideName, 
                                                ffcSB.toString());
                                
                                continue;
                        }
                        
                        Map<Integer, String> featureLabels = 
                                this.processFeature(allLines, slideDef, 
                                        featureLineNumberRangePerSlide.get(slideName));
                        
                        // Find feature definition IDs
                        for(String featureLabel : featureLabels.values()) {
                                FeatureDefinition featureDef = defs.getFeatureDefinition(
                                                slideID.longValue(), featureLabel);
                                
                                this.slideNameFeatureNameToFeatureDefID.put(
                                                (slideName + "_" + featureLabel), 
                                                new Long(featureDef.getID()));
                        }
                        
                        slideFeatureLabels.put(slideName, featureLabels);
                        this.slideNameToFeatureLabels.put(slideName, 
                                        featureLabels.values());
                }
                
                Utils.logger.info("slideFeatureLabels=" + slideFeatureLabels);
                Utils.logger.info("slideNameToFeatureLabels=" + 
                                this.slideNameToFeatureLabels);
                
                Utils.logger.info("slideNameToSlideDefID=" + 
                                this.slideNameToSlideDefID);
                
                Utils.logger.info("slideNameFeatureNameToFeatureDefID=" + 
                                this.slideNameFeatureNameToFeatureDefID);
                
                // 7 Bullets
                // Collect all bullets
                for(String slideName : slideFeatureLabels.keySet()) {
                        Map<Integer, String> featureLabels = 
                                slideFeatureLabels.get(slideName);
                        
                        if(featureLabels == null || featureLabels.size() == 0) {
                                continue;
                        }
                        
                        // 7.1 Find line number range between which content for bullets 
                        // is found (per feature on this slide)
                        Map<String, int[]> bulletLineNumberRangePerFeature = 
                                this.getBulletLineRange(allLines, featureLabels);
                        
                        for(String featureLabel : bulletLineNumberRangePerFeature.keySet()){

                                String key = slideName + "_" + featureLabel;
                                Long featureDefID = 
                                        this.slideNameFeatureNameToFeatureDefID.get(key);
                                FeatureDefinition featureDef = 
                                        defs.getFeatureDefinition(featureDefID.longValue());
                                
                                Long slideDefID = this.slideNameToSlideDefID.get(slideName);
                                SlideDefinition slideDef = defs.getSlideDefinition(slideDefID);
                                
                                this.processBullets(allLines, 
                                                bulletLineNumberRangePerFeature.get(featureLabel), 
                                                featureDef, slideDef);
                        }
                }
                
                // Construct Content Model
                Entity entity = this.contructEntityObject();
                Utils.logger.info("entity="+entity);
                
                EntityContainer entContainer = 
                        Catalog.getInstance().getEntityContainer();
                
                entContainer.addEntity(entity);
                
                // RE-VISIT
                // Store it back
                DBUtils.store(entContainer.getEntities(), this.dbFile);
                
                // Store the index separately
                String entitiesbycategoryDBFile = CN.CONTENT_DB_PATH + 
                        "entitiesbycategory" + ".ser";
                
                DBUtils.store(entContainer.getEntitiesbyCategory(), 
                                entitiesbycategoryDBFile);
        }
        
        /**
         * 
         * @return Entity 
         * @throws Exception 
         */
        private Entity contructEntityObject() throws Exception {
                SequenceGenerator sg = SequenceGenerator.getInstance();
                long entityID = sg.getNextSequence(SequenceGenerator.ENTITY);
                
                Entity entity = new Entity(entityID, this.categoryID);
                String brandModel[] = StringUtils.split(this.introductionTitle, " ");
                
                if(brandModel.length == 1) {
                        entity.setBrand(StringUtils.strip(brandModel[0]));
                }
                else if(brandModel.length == 2) {
                        entity.setBrand(StringUtils.strip(brandModel[0]));
                        entity.setModelName(StringUtils.strip(brandModel[1]));
                }
                else if(brandModel.length == 3) {
                        entity.setBrand(StringUtils.strip(brandModel[0]));
                        entity.setModelNumber(StringUtils.strip(brandModel[1]));
                        entity.setModelName(StringUtils.strip(brandModel[2]));
                }
                else if(brandModel.length == 4) {
                        entity.setBrand(StringUtils.strip(brandModel[0]) + " " + 
                                        StringUtils.strip(brandModel[1]));
                        
                        entity.setModelNumber(StringUtils.strip(brandModel[2]));
                        entity.setModelName(StringUtils.strip(brandModel[3]));
                }
                        
                for(String slideName : this.slideNames) {
                        Collection<String> featureNames = 
                                this.slideNameToFeatureLabels.get(slideName);
                        
                        Long slideDefID = this.slideNameToSlideDefID.get(slideName);
                        Slide slide = new Slide(slideDefID.longValue());
                        
                        // Only if there are features
                        if(featureNames != null) {
                                List<Feature> features = new ArrayList<Feature>();
                                
                                for(String featureName : featureNames) {
                                        Long featureDefID = this.slideNameFeatureNameToFeatureDefID.get(
                                                        slideName + "_" + featureName);
                                        
                                        List<Bullet> bullets = this.slideNameFeatureNameToBullets.get(
                                                        slideName + "_" + featureName);
                                        
                                        Feature feature = new Feature(featureDefID.longValue());
                                        feature.setBullets(bullets);
                                        
                                        features.add(feature);
                                }
                                
                                slide.setFeatures(features);
                        }
                        
                        // Add free-form content is collect above
                        String ffc = this.slideNameToFreeformContent.get(slideName);
                        Utils.logger.info("slideName=" + slideName + " ffc=" + ffc);
                        
                        if(ffc != null) {
                                slide.setFreeformContent(new FreeformContent(ffc));
                        }
                        
                        entity.addSlide(slide);
                }
                
                return entity;
        }
        
        /**
         * 
         * @param allLines
         * @param bulletLineNumberRange
         * @param featureDef
         * @throws Exception 
         */
        private void processBullets(List<String> allLines, 
                        int[] bulletLineNumberRange, FeatureDefinition featureDef, 
                        SlideDefinition slideDef) throws Exception {
                
                int from = bulletLineNumberRange[0];
                int to = bulletLineNumberRange[1];
                
                BulletDefinition bulletDef = featureDef.getBulletDefinition();
                Utils.logger.info("bulletDefinition=" + bulletDef);
                
                DefinitionsContainer defs = 
                        Catalog.getInstance().getDefinitionsContainer();
                
                String crumb = "\"" + slideDef.getLabel() + "\" > \"" + 
                        featureDef.getLabel() + "\"";
                
                // Enforce isMultivalue
                // RE-VISIT - This validation should be inside for loop below
                // Skipping for now this may not be a common error
                /*
                if(!bulletDef.isMultivalue() && (from != to)) {
                        Utils.logger.severe("Only one bullet is allowed for " + crumb);
                        return;
                }
                */

                // If unit is defined
                Unit unitDef =  null;
                if(bulletDef.getUnitID() != 0L) {
                        long unitID = bulletDef.getUnitID();
                        unitDef = defs.getUnit(unitID);
                        Utils.logger.info("unitDef=" + unitDef);
                }

                long datatypeDefID = bulletDef.getDatatypeDefinitionID();
                DatatypeDefinition datatypeDef = 
                        defs.getDatatypeDefinition(datatypeDefID);
                Utils.logger.info("datatypeDef=" + datatypeDef);

                
                String key = StringUtils.lowerCase(slideDef.getLabel()) + "_" + 
                        StringUtils.lowerCase(featureDef.getLabel());
                
                List<Bullet> bullets = 
                        this.slideNameFeatureNameToBullets.get(key);
                
                if(bullets == null) {
                        bullets = new ArrayList<Bullet>();
                        this.slideNameFeatureNameToBullets.put(key, bullets);
                }
                
                // If primitive
                boolean isEnum = false;
                boolean isComposite = false;
                boolean isPrimitive = false;
                if(datatypeDef instanceof EnumDefinition) {
                        isEnum = true;
                }
                else if(datatypeDef instanceof CompositeDefinition) {
                        isComposite = true;
                }
                else {
                        isPrimitive = true;
                }
                
                Utils.logger.info("isEnum=" + isEnum + " isComposite=" + isComposite +
                                " isPrimitive=" + isPrimitive);
                
                Utils.logger.info("from=" + from + " to=" + to);
                for(int i=from; i<to+1; i++) {
                        String line = allLines.get(i);
                        Utils.logger.info("Line " + i + ": " + line);

                        // VALIDATE
                        // It means free-form content formatted at feature level
                        // Assume there are no more bullets
                        if(StringUtils.startsWith(line, CN.PATTERN_LEVEL_1)) {
                                break;
                        }
                        
                        if(!StringUtils.startsWith(line, CN.PATTERN_LEVEL_2)) {
                                continue;
                        }
                        
                        String bulletText = StringUtils.strip(StringUtils.stripStart(line, 
                                        CN.PATTERN_LEVEL_2));
                        Utils.logger.info("bulletText=" + bulletText);
                        
                        // if no unit is defined for this bullet whole is treated as value
                        String bulletValue = bulletText;
                        
                        // If unit is defined
                        if(bulletDef.getUnitID() != 0L) {
                                
                                // Validate unit
                                String[] parts = StringUtils.split(bulletText, " ");
                                if(parts.length < 2) {
                                        Utils.logger.severe("Unit is missing, " + crumb + " = " + 
                                                        bulletText);
                                        
                                        continue;
                                }
                                
                                if(parts.length > 2) {
                                        Utils.logger.severe("Invalid value, " + crumb + " = " + 
                                                        bulletText);
                                        
                                        continue;
                                }
                                
                                bulletValue = parts[0];
                                String unitValue = parts[1];
                                Utils.logger.info("unitValue="+unitValue);
                                
                                if(!(unitValue.equalsIgnoreCase(unitDef.getShortForm()) ||
                                                unitValue.equalsIgnoreCase(unitDef.getFullForm()))) {
                                        Utils.logger.severe("Invalid unit, " + crumb + " = " + 
                                                        bulletText);
                                        
                                        continue;
                                }
                        }
                        
                        // Validate bullet value
                        Utils.logger.info("bulletValue=" + bulletValue);
                        if(isPrimitive) {
                                if(!this.validatePrimitive(bulletValue, datatypeDef, crumb)) {
                                        continue;
                                }
                                
                                bullets.add(new Bullet(new PrimitiveDataObject(bulletValue)));
                        }
                        
                        // Enum and fixed
                        else if(isEnum && !bulletDef.isLearned()) {
                                long enumValueID = defs.getEnumValueID(datatypeDef.getID(), 
                                                bulletValue);
                                Utils.logger.info("enumValueID=" + enumValueID);
                                
                                // Treat it to be free-form
                                if(enumValueID == -1L) {
                                        Utils.logger.warning("Not one of the valid enum values, " +
                                                        "treating it like free-form content - " + crumb + 
                                                        " = " + bulletValue);
                                        
                                        continue;
                                }
                                
                                EnumDataObject enumDataObject = new EnumDataObject(enumValueID);
                                bullets.add(new Bullet(enumDataObject));
                        }
                        
                        // Composite
                        else if(isComposite) {
                                CompositeDefinition compositeDef = 
                                        (CompositeDefinition)datatypeDef;
                                
                                String separator = compositeDef.getSeparator();
                                String[] compositeParts = 
                                        StringUtils.split(bulletValue, separator);
                                
                                List<CompositePartDefinition> compositePartDefs = 
                                        compositeDef.getCompositePartDefinitions();
                                
                                // Validate number of parts
                                if(compositeParts.length != compositePartDefs.size()) {
                                        Utils.logger.severe("Invalid value, " + crumb + " = " + 
                                                        bulletValue);
                                        
                                        continue;
                                }
                                
                                // Remove spurious whitespaces
                                boolean validPart = true;
                                for(int j=0;j<compositeParts.length;j++) {
                                        compositeParts[j] = StringUtils.strip(compositeParts[j]);
                                        Utils.logger.info("compositeParts[j]=" + compositeParts[j]);
                                        
                                        // Validate each part
                                        // Each part can be enum or composite in itself
                                        // We will stick to primitive for now
                                        long partDatatypeDefID = 
                                                compositePartDefs.get(j).getDatatypeDefinitionID();
                                        
                                        DatatypeDefinition partDatatypeDef = 
                                                defs.getDatatypeDefinition(partDatatypeDefID);
                                        Utils.logger.info("partDatatypeDef=" + partDatatypeDef);
                                        
                                        if(!this.validatePrimitive(compositeParts[j], 
                                                        partDatatypeDef, crumb)) {
                                                validPart = false;
                                                break;
                                        }
                                }
                                
                                if(!validPart) {
                                        continue;
                                }

                                CompositeDataObject compositeDataObject = 
                                        new CompositeDataObject();

                                for(int j=0;j<compositeParts.length;j++) {
                                        compositeDataObject.addPrimitiveDataObject(
                                                        new PrimitiveDataObject(compositeParts[j]));
                                }
                                
                                bullets.add(new Bullet(compositeDataObject));
                        }
                }
                
                Utils.logger.info("slideNameFeatureNameToBullets=" + 
                                this.slideNameFeatureNameToBullets);
        }
        
        /**
         * 
         * @param bulletValue
         * @param datatypeDef
         * @param crumb
         */
        private boolean validatePrimitive(String bulletValue, 
                        DatatypeDefinition datatypeDef, String crumb) {
                String dt = datatypeDef.getName();
                boolean valid = true;
                
                // integer
                if(dt.equals("integer")) {
                        try {
                                Integer.parseInt(bulletValue);
                        }
                        catch(NumberFormatException nfe) {
                                Utils.logger.severe("Invalid integer value, " + crumb +
                                                " = " + bulletValue);
                                valid = false;
                        }
                }
                
                // decimal
                else if(dt.endsWith("decimal")) {
                        try {
                                Float.parseFloat(bulletValue);
                        }
                        catch(NumberFormatException nfe) {
                                Utils.logger.severe("Invalid decimal value, " + crumb +
                                                " = " + bulletValue);
                                valid = false;
                        }
                }
                
                return valid;
        }
        
        // string
        // any thing goes
        /**
         * 
         * @param allLines
         * @param featureLabels
         * @return
         */
        private Map<String, int[]> getBulletLineRange(List<String> allLines, 
                        Map<Integer, String> featureLabels) {
                Utils.logger.info("allLines=" + allLines);
                Utils.logger.info("featureLabels=" + featureLabels);
                
                // 5.1 Find line number range between which content for feature is found
                Map<String, int[]> bulletLineNumberRangePerFeature = 
                        new HashMap<String, int[]>();
                        
                Utils.logger.info("featureLabels.keySet=" + 
                                featureLabels.keySet().toString());
                
                Integer[] featureLineNumbers = featureLabels.keySet().toArray(
                                new Integer[0]);
                
                for(int i=0; i<featureLineNumbers.length-1; i++) {
                        String tFeatureLabel = featureLabels.get(
                                        new Integer(featureLineNumbers[i]));
                        
                        Utils.logger.info("tFeatureLabel=" + tFeatureLabel);
                        
                        int from = featureLineNumbers[i]+1;
                        int to = featureLineNumbers[i+1]-1;
                        Utils.logger.info("from=" + from + " to=" + to);
                        
                        bulletLineNumberRangePerFeature.put(tFeatureLabel, 
                                        new int[] {from, to});
                }

                // For last feature
                String lastFeatureLabel = featureLabels.get(
                        new Integer(featureLineNumbers[featureLineNumbers.length-1]));
                Utils.logger.info("lastFeatureLabel=" + lastFeatureLabel);
                
                int lastFrom = featureLineNumbers[featureLineNumbers.length-1] + 1;
                Utils.logger.info("lastFrom=" + lastFrom);
                
                // First empty line from "lastFrom"
                int lastTo = lastFrom;
                while(true) {
                        if(lastTo == (allLines.size() - 1)) {
                                break;
                        }
                        
                        String tLastLine = allLines.get(lastTo + 1);
                        Utils.logger.info("tLastLine=" + tLastLine);
                        if(tLastLine == null) {
                                break;
                        }
                        
                        if(!StringUtils.startsWith(tLastLine, CN.PATTERN_LEVEL_2)) {
                                break;
                        }
                        lastTo++;
                        Utils.logger.info("running lastTo=" + lastTo);
                }
                
                Utils.logger.info("lastFrom=" + lastFrom + " lastTo=" + lastTo);
                        
                bulletLineNumberRangePerFeature.put(lastFeatureLabel, 
                                new int[] {lastFrom, lastTo});
                
                Utils.logger.info("featureLineNumberRangePerSlide=" + 
                                bulletLineNumberRangePerFeature.toString());            
                
                return bulletLineNumberRangePerFeature;
        }

        
        /**
         * 
         * @param allLines
         * @param slideDef
         * @param lineNumberRange
         * @throws Exception
         */
        private Map<Integer, String> processFeature(List<String> allLines, 
                        SlideDefinition slideDef, 
                        int[] lineNumberRange) throws Exception {
                
                int from = lineNumberRange[0];
                int to = lineNumberRange[1];
                
                Utils.logger.info("from=" + from + " to=" + to);
                
                int numberOfLines = (to - from);
                Utils.logger.info("numberOfLines=" + numberOfLines);

                Map<Integer, String> candidateFeatures = new TreeMap<Integer, String>();
                for(int i=from; i<(to+1); i++) {
                        String line = allLines.get(i);

                        if(StringUtils.startsWith(line, CN.PATTERN_LEVEL_1)) {
                                //Utils.logger.info(line);
                                //Utils.logger.info("Level 1");
                                candidateFeatures.put(new Integer(i), 
                                                StringUtils.strip(StringUtils.strip(line, 
                                                                CN.PATTERN_LEVEL_1)));
                        }
                }
                
                Utils.logger.info("candidateFeatures=" + candidateFeatures);
                
                // Get all feature definitions for the slide
                DefinitionsContainer defs = 
                        Catalog.getInstance().getDefinitionsContainer();
                
                Utils.logger.info("slideDef.getID=" + slideDef.getID());
                
                List<FeatureDefinition> featureDefs = 
                        defs.getFeatureDefinitions(slideDef.getID());
                Utils.logger.info("featureDefs=" + featureDefs);
                
                List<String> validFeatureLabels = new ArrayList<String>();
                for(int i=0; i<featureDefs.size(); i++) {
                        validFeatureLabels.add(
                                        StringUtils.lowerCase(featureDefs.get(i).getLabel()));
                }
                Utils.logger.info("validFeatureLabels=" + validFeatureLabels);
                
                // Filter valid features
                Map<Integer, String> featureLabels = new TreeMap<Integer, String>();

                // Discard if feature is not one of the valids
                // RE-VISIT
                for(Integer lineNumber : candidateFeatures.keySet()) {
                        if(validFeatureLabels.contains(StringUtils.lowerCase(
                                        candidateFeatures.get(lineNumber)))) {
                                featureLabels.put(lineNumber, StringUtils.lowerCase(
                                                candidateFeatures.get(lineNumber)));
                        }
                }
                Utils.logger.info("featureLabels=" + featureLabels);
                
                // Fetch all mandatory features
                List<FeatureDefinition> mandatoryFeatureDefs = 
                        defs.getFeatureDefinitions(slideDef.getID(),
                        EditorialImportance.MANDATORY);

                Utils.logger.info("mandatoryFeatureDefs=" + mandatoryFeatureDefs);
                
                // Severe error if mandatory features are not included
                for(FeatureDefinition featureDef : mandatoryFeatureDefs) {
                        if(!featureLabels.values().contains(StringUtils.lowerCase(
                                        featureDef.getLabel()))) {
                                Utils.logger.severe("Mandatory feature \"" + slideDef.getLabel() 
                                                + " > " + featureDef.getLabel() + "\" is missing");
                        }
                }
                
                // Fetch all recommended features
                List<FeatureDefinition> recommendedFeatureDefs = 
                        defs.getFeatureDefinitions(slideDef.getID(),
                        EditorialImportance.RECOMMENDED);

                Utils.logger.info("recommendedFeatureDefs=" + recommendedFeatureDefs);
                
                // Warn if recommended features are not included
                for(FeatureDefinition featureDef : recommendedFeatureDefs) {
                        if(!featureLabels.values().contains(StringUtils.lowerCase(
                                        featureDef.getLabel()))) {
                                Utils.logger.warning("Recommended feature \"" + 
                                                featureDef.getLabel() + "\" is missing");
                        }
                }
                
                return featureLabels;
        }
        
        /**
         * Filter and validate parent slide names
         * 
         * @param candidateSlideNames
         * @return List<LineInfo>
         * @throws Exception 
         */
        private Map<Integer, String> processSlides(
                        Map<Integer, String> candidateSlideLines) throws Exception {
                
                // Order of keys need to be restored
                Map<Integer, String> processedSlideLines = 
                        new TreeMap<Integer, String>();
                
                // 1 Retrieve meta-data
                // 1.1 Retrieve all valid slide names in the content model
                DefinitionsContainer defs = 
                        Catalog.getInstance().getDefinitionsContainer();
                
                List<SlideDefinition> slideDefs = defs.getSlideDefinitions(
                                this.categoryID);
                
                List<String> validSlideNames = new ArrayList<String>();
                for (SlideDefinition slideDef : slideDefs) {
                        
                        // To avoid Introduction slide
                        if(!slideDef.getLabel().isEmpty()) {
                                validSlideNames.add(StringUtils.lowerCase(slideDef.getLabel()));
                        }
                }
                Utils.logger.info("validSlideNames=" + validSlideNames.toString());
                
                // 2 Rules
                // 2.1 Discard if slide is not one of the valids
                // REVISIT
                for(Integer lineNumber : candidateSlideLines.keySet()) {
                        if(validSlideNames.contains(StringUtils.lowerCase(
                                        candidateSlideLines.get(lineNumber)))) {
                                
                                processedSlideLines.put(lineNumber, StringUtils.lowerCase(
                                                                candidateSlideLines.get(lineNumber)));
                                
                                this.slideNames.add(StringUtils.lowerCase(
                                                candidateSlideLines.get(lineNumber)));
                        }
                }
                
                Utils.logger.info("processedSlideLines=" + 
                                processedSlideLines.toString());

                // 3 Retrieve "Mandatory" slide names for the category
                List<SlideDefinition> mandatorySlideDefs = 
                        defs.getSlides(this.categoryID, EditorialImportance.MANDATORY);
                
                Utils.logger.info("mandatorySlideDefs=" + 
                                mandatorySlideDefs.toString());

                // 3.1 All mandatory slides exist - Severe
                for(SlideDefinition mandatorySlideDef : mandatorySlideDefs) {
                        
                        // Avoid introduction slide
                        if(mandatorySlideDef.getLabel().isEmpty()) {
                                continue;
                        }
                        
                        if(!processedSlideLines.values().contains(
                                        StringUtils.lowerCase(mandatorySlideDef.getLabel()))) {
                                Utils.logger.severe("Mandatory slide \"" + 
                                                mandatorySlideDef.getLabel() + "\" is missing");
                        }
                }

                // 4 Retrieve "Recommended" slide names for the category
                List<SlideDefinition> recommendedSlideDefs = 
                        defs.getSlides(this.categoryID, EditorialImportance.RECOMMENDED);

                Utils.logger.info("recommendedSlideDefs=" + 
                                recommendedSlideDefs.toString());
                
                // 4.1 All recommended slides exist - Warn      
                for(SlideDefinition recommendedSlideDef : recommendedSlideDefs) {
                        if(!processedSlideLines.values().contains(
                                        StringUtils.lowerCase(recommendedSlideDef.getLabel()))) {
                                Utils.logger.warning("Recommended slide \"" + 
                                                recommendedSlideDef.getLabel() + "\" is missing");
                        }
                }
                
                return processedSlideLines;
        }
        
        /**
         * @throws Exception 
         * 
         */
        @SuppressWarnings("unused")
        private void test() throws Exception {
                // test category
                DefinitionsContainer allDefs = 
                        Catalog.getInstance().getDefinitionsContainer();
                
                Category category = allDefs.getCategory(this.categoryID);
                Utils.logger.info(category.toString());
                
                // test data type defs
                DatatypeDefinition datatypeDef = allDefs.getDatatypeDefinition(70004L);
                Utils.logger.info(datatypeDef.toString());
                
                // test units
                Unit unit = allDefs.getUnit(50004L);
                Utils.logger.info(unit.toString());
                
                // test slide defs
                SlideDefinition slideDef = allDefs.getSlideDefinition(130002L);
                Utils.logger.info(slideDef.toString());
                
                // test feature defs
                FeatureDefinition featureDef = allDefs.getFeatureDefinition(120002L);
                Utils.logger.info(featureDef.toString());
        }
}