Subversion Repositories SmartDukaan

package com.smartdukaan.cron.offercircular;

import java.io.File;

import java.util.ArrayList;

import java.util.List;

import java.util.regex.Matcher;

import java.util.regex.Pattern;

import org.apache.logging.log4j.LogManager;

import org.apache.logging.log4j.Logger;

import org.apache.pdfbox.pdmodel.PDDocument;

import org.springframework.stereotype.Component;

import org.apache.pdfbox.text.PDFTextStripper;

import technology.tabula.ObjectExtractor;

import technology.tabula.Page;

import technology.tabula.RectangularTextContainer;

import technology.tabula.Table;

import technology.tabula.extractors.SpreadsheetExtractionAlgorithm;

/**

 * Reads the nine verbatim columns out of a Pine Labs affordability circular PDF.

 *

 * Verified against the reference Python implementation (pdfplumber) on the Aug'26

 * circular: 324 rows, six middle cells each, zero field differences.

 */

@Component

public class CircularExtractor {

    private static final Logger LOGGER = LogManager.getLogger(CircularExtractor.class);

    /** Every data row ends with two dd-MM-yyyy cells; that pair is the anchor. */

    private static final Pattern DATE = Pattern.compile("^\\d{2}-\\d{2}-\\d{4}$");

    /** A "# ..." note printed under the table, with the page it applies to. */

    public static class Footnote {

        private final int pageNo;

        private final String text;

        Footnote(int pageNo, String text) { this.pageNo = pageNo; this.text = text; }

        public int getPageNo() { return pageNo; }

        public String getText() { return text; }

        @Override public String toString() { return "p" + pageNo + ": " + text; }

    }

    public static class Result {

        private final List<CircularRow> rows = new ArrayList<>();

        private final List<Footnote> footnotes = new ArrayList<>();

        private final List<String> warnings = new ArrayList<>();

        public List<CircularRow> getRows() { return rows; }

        public List<Footnote> getFootnotes() { return footnotes; }

        public List<String> getWarnings() { return warnings; }

    }

    /** Footnote lines start with '#' in the page text below the table. */

    private static final Pattern FOOTNOTE = Pattern.compile("#\\s*(.+)");

    public Result extract(File pdf) throws Exception {

        Result result = new Result();

        try (PDDocument document = PDDocument.load(pdf)) {

            ObjectExtractor objects = new ObjectExtractor(document);

            SpreadsheetExtractionAlgorithm lattice = new SpreadsheetExtractionAlgorithm();

            for (int pageNo = 1; pageNo <= document.getNumberOfPages(); pageNo++) {

                Page page = objects.extract(pageNo);

                // tabula returns several overlapping candidates per page - a

                // page-level bounding box plus the real grid. Keep whichever yields

                // the most data rows. Do NOT dedupe by row content: the circular

                // legitimately repeats identical rows, and content-deduping silently

                // discards them.

                List<CircularRow> best = new ArrayList<>();

                for (Table table : lattice.extract(page)) {

                    List<CircularRow> candidate = readTable(pageNo, table, result);

                    if (candidate.size() > best.size()) {

                        best = candidate;

                    }

                }

                if (best.isEmpty()) {

                    result.getWarnings().add("page " + pageNo + ": no data rows found");

                }

                LOGGER.debug("circular page {} yielded {} data rows", pageNo, best.size());

                result.getRows().addAll(best);

                result.getFootnotes().addAll(readFootnotes(document, pageNo));

            }

        }

        LOGGER.info("extracted {} circular rows and {} footnote(s) from {} ({} warning(s))",

                result.getRows().size(), result.getFootnotes().size(), pdf.getName(),

                result.getWarnings().size());

        return result;

    }

    /**

     * Footnotes sit in the page text BELOW the table, so tabula cannot see them -

     * they need a plain text extraction of the same page.

     */

    private List<Footnote> readFootnotes(PDDocument document, int pageNo) throws Exception {

        PDFTextStripper stripper = new PDFTextStripper();

        stripper.setStartPage(pageNo);

        stripper.setEndPage(pageNo);

        List<Footnote> notes = new ArrayList<>();

        Matcher m = FOOTNOTE.matcher(stripper.getText(document));

        while (m.find()) {

            String text = m.group(1).trim();

            if (!text.isEmpty()) {

                notes.add(new Footnote(pageNo, text));

            }

        }

        return notes;

    }

    private List<CircularRow> readTable(int pageNo, Table table, Result result) {

        List<CircularRow> rows = new ArrayList<>();

        int rowNo = 0;

        for (List<RectangularTextContainer> raw : table.getRows()) {

            List<String> cells = compact(raw);

            if (cells.size() < 4) {

                continue;

            }

            String start = cells.get(cells.size() - 2);

            String end = cells.get(cells.size() - 1);

            if (!DATE.matcher(start).matches() || !DATE.matcher(end).matches()) {

                continue;                        // header, footnote or spacer row

            }

            List<String> middle = new ArrayList<>(cells.subList(1, cells.size() - 2));

            if (middle.size() != CircularRow.MIDDLE_CELL_COUNT) {

                result.getWarnings().add("page " + pageNo + ": row with "

                        + middle.size() + " middle cells (expected "

                        + CircularRow.MIDDLE_CELL_COUNT + "): " + cells);

                continue;

            }

            rows.add(new CircularRow(pageNo, ++rowNo, cells.get(0), middle, start, end));

        }

        return rows;

    }

    /**

     * Flattens a tabula row to non-empty, whitespace-normalised strings.

     *

     * Every empty cell is dropped, not just trailing ones. Pages carry different

     * numbers of vertical rulings (page 8 of the Aug'26 circular has 48 against 34

     * elsewhere), which interleaves blank cells between the real ones - including

     * between the two date cells, which defeats any positional anchor. Empty cells

     * are never meaningful in this document: after compaction a data row is exactly

     * OEM label + six middle cells + two dates.

     *

     * Newlines are replaced with spaces because they are word-wrap points, not

     * structure - they fall mid-product ("iPhone 17" / "Pro Max") and mid-variant

     * ("(12/256GB)" / "(12/512GB)").

     */

    private List<String> compact(List<RectangularTextContainer> raw) {

        List<String> cells = new ArrayList<>();

        for (RectangularTextContainer cell : raw) {

            String text = cell.getText() == null ? "" : cell.getText();

            text = text.replace('\r', ' ').replace('\n', ' ').trim().replaceAll("\\s+", " ");

            if (!text.isEmpty()) {

                cells.add(text);

            }

        }

        return cells;

    }

}