Subversion Repositories SmartDukaan

package com.smartdukaan.cron.offercircular;

import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.springframework.stereotype.Component;

import org.apache.pdfbox.text.PDFTextStripper;

import technology.tabula.ObjectExtractor;
import technology.tabula.Page;
import technology.tabula.RectangularTextContainer;
import technology.tabula.Table;
import technology.tabula.extractors.SpreadsheetExtractionAlgorithm;

/**
 * Reads the nine verbatim columns out of a Pine Labs affordability circular PDF.
 *
 * Verified against the reference Python implementation (pdfplumber) on the Aug'26
 * circular: 324 rows, six middle cells each, zero field differences.
 */
@Component
public class CircularExtractor {

    private static final Logger LOGGER = LogManager.getLogger(CircularExtractor.class);

    /** Every data row ends with two dd-MM-yyyy cells; that pair is the anchor. */
    private static final Pattern DATE = Pattern.compile("^\\d{2}-\\d{2}-\\d{4}$");

    /** A "# ..." note printed under the table, with the page it applies to. */
    public static class Footnote {
        private final int pageNo;
        private final String text;
        Footnote(int pageNo, String text) { this.pageNo = pageNo; this.text = text; }
        public int getPageNo() { return pageNo; }
        public String getText() { return text; }
        @Override public String toString() { return "p" + pageNo + ": " + text; }
    }

    public static class Result {
        private final List<CircularRow> rows = new ArrayList<>();
        private final List<Footnote> footnotes = new ArrayList<>();
        private final List<String> warnings = new ArrayList<>();
        public List<CircularRow> getRows() { return rows; }
        public List<Footnote> getFootnotes() { return footnotes; }
        public List<String> getWarnings() { return warnings; }
    }

    /** Footnote lines start with '#' in the page text below the table. */
    private static final Pattern FOOTNOTE = Pattern.compile("#\\s*(.+)");

    public Result extract(File pdf) throws Exception {
        Result result = new Result();
        try (PDDocument document = PDDocument.load(pdf)) {
            ObjectExtractor objects = new ObjectExtractor(document);
            SpreadsheetExtractionAlgorithm lattice = new SpreadsheetExtractionAlgorithm();

            for (int pageNo = 1; pageNo <= document.getNumberOfPages(); pageNo++) {
                Page page = objects.extract(pageNo);

                // tabula returns several overlapping candidates per page - a
                // page-level bounding box plus the real grid. Keep whichever yields
                // the most data rows. Do NOT dedupe by row content: the circular
                // legitimately repeats identical rows, and content-deduping silently
                // discards them.
                List<CircularRow> best = new ArrayList<>();
                for (Table table : lattice.extract(page)) {
                    List<CircularRow> candidate = readTable(pageNo, table, result);
                    if (candidate.size() > best.size()) {
                        best = candidate;
                    }
                }
                if (best.isEmpty()) {
                    result.getWarnings().add("page " + pageNo + ": no data rows found");
                }
                LOGGER.debug("circular page {} yielded {} data rows", pageNo, best.size());
                result.getRows().addAll(best);
                result.getFootnotes().addAll(readFootnotes(document, pageNo));
            }
        }
        LOGGER.info("extracted {} circular rows and {} footnote(s) from {} ({} warning(s))",
                result.getRows().size(), result.getFootnotes().size(), pdf.getName(),
                result.getWarnings().size());
        return result;
    }

    /**
     * Footnotes sit in the page text BELOW the table, so tabula cannot see them -
     * they need a plain text extraction of the same page.
     */
    private List<Footnote> readFootnotes(PDDocument document, int pageNo) throws Exception {
        PDFTextStripper stripper = new PDFTextStripper();
        stripper.setStartPage(pageNo);
        stripper.setEndPage(pageNo);
        List<Footnote> notes = new ArrayList<>();
        Matcher m = FOOTNOTE.matcher(stripper.getText(document));
        while (m.find()) {
            String text = m.group(1).trim();
            if (!text.isEmpty()) {
                notes.add(new Footnote(pageNo, text));
            }
        }
        return notes;
    }

    private List<CircularRow> readTable(int pageNo, Table table, Result result) {
        List<CircularRow> rows = new ArrayList<>();
        int rowNo = 0;
        for (List<RectangularTextContainer> raw : table.getRows()) {
            List<String> cells = compact(raw);
            if (cells.size() < 4) {
                continue;
            }
            String start = cells.get(cells.size() - 2);
            String end = cells.get(cells.size() - 1);
            if (!DATE.matcher(start).matches() || !DATE.matcher(end).matches()) {
                continue;                        // header, footnote or spacer row
            }
            List<String> middle = new ArrayList<>(cells.subList(1, cells.size() - 2));
            if (middle.size() != CircularRow.MIDDLE_CELL_COUNT) {
                result.getWarnings().add("page " + pageNo + ": row with "
                        + middle.size() + " middle cells (expected "
                        + CircularRow.MIDDLE_CELL_COUNT + "): " + cells);
                continue;
            }
            rows.add(new CircularRow(pageNo, ++rowNo, cells.get(0), middle, start, end));
        }
        return rows;
    }

    /**
     * Flattens a tabula row to non-empty, whitespace-normalised strings.
     *
     * Every empty cell is dropped, not just trailing ones. Pages carry different
     * numbers of vertical rulings (page 8 of the Aug'26 circular has 48 against 34
     * elsewhere), which interleaves blank cells between the real ones - including
     * between the two date cells, which defeats any positional anchor. Empty cells
     * are never meaningful in this document: after compaction a data row is exactly
     * OEM label + six middle cells + two dates.
     *
     * Newlines are replaced with spaces because they are word-wrap points, not
     * structure - they fall mid-product ("iPhone 17" / "Pro Max") and mid-variant
     * ("(12/256GB)" / "(12/512GB)").
     */
    private List<String> compact(List<RectangularTextContainer> raw) {
        List<String> cells = new ArrayList<>();
        for (RectangularTextContainer cell : raw) {
            String text = cell.getText() == null ? "" : cell.getText();
            text = text.replace('\r', ' ').replace('\n', ' ').trim().replaceAll("\\s+", " ");
            if (!text.isEmpty()) {
                cells.add(text);
            }
        }
        return cells;
    }
}