Subversion Repositories SmartDukaan

package com.spice.profitmandi.web.services;

import java.util.Arrays;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.Set;

/**
 * Near-duplicate detection between the remarks of ONE checkout.
 *
 * Why this lives here and not in the analyze service: that service compares a
 * remark only against the ASM's OTHER visits, and matches on normalized-exact
 * text (every hit comes back at similarity 1.0). Both of those miss the case the
 * checkout form actively provokes — the app rejects a partial fill, so a rep with
 * five open agendas and one thing to say pastes it into all five boxes, lightly
 * reworded. Same visit_id, non-identical text: invisible upstream.
 *
 * Content-word overlap (Jaccard) is what we can do without an embedding model.
 * It catches the copy-with-tweaks family — punctuation and filler edits, word
 * reordering, moderate rewording — and it does NOT catch a genuine paraphrase
 * built from different vocabulary. Measured against the same seed remark:
 *
 *   "…collected 45,000 … will place the order on Monday."   0.85  flagged
 *   "Owner met, collected 45000 towards old dues, order …"  0.64  flagged
 *   "Recovered pending payment of 45k … ordering Monday."    0.10  missed
 *
 * Closing that last gap needs sentence embeddings, i.e. an analyze-side change.
 */
public final class RemarkSimilarity {

    private RemarkSimilarity() {}

    /** Overlap at or above this counts as the same remark reused. */
    public static final double DUPLICATE_THRESHOLD = 0.6;

    /**
     * Below this many content words a remark is too thin to compare — two such
     * remarks overlap trivially, and the scorer already rejects them outright
     * (TOO_SHORT / SINGLE_WORD).
     */
    private static final int MIN_CONTENT_WORDS = 3;

    /** Filler with no bearing on what was discussed; English + common Hinglish. */
    private static final Set<String> STOPWORDS = new HashSet<>(Arrays.asList(
            "a", "an", "the", "and", "or", "but", "also", "as", "at", "by", "for", "from", "in", "into",
            "of", "on", "to", "with", "is", "was", "were", "be", "been", "am", "are", "will", "would",
            "has", "have", "had", "do", "did", "does", "done", "not", "no", "he", "she", "it", "they",
            "we", "i", "his", "her", "their", "our", "my", "them", "him", "that", "this", "these",
            "those", "there", "then", "so", "very", "today", "ka", "ke", "ki", "ko", "kaa", "hai",
            "tha", "the2", "se", "me", "mein", "aur", "nahi", "bhi", "kar", "karke", "kiya", "diya"));

    /**
     * Jaccard overlap of the two remarks' content words, 0..1. Returns 0 when
     * either side is too thin to judge, so callers can treat any positive result
     * as comparable.
     */
    public static double similarity(String a, String b) {
        Set<String> tokensA = contentWords(a);
        Set<String> tokensB = contentWords(b);
        if (tokensA.size() < MIN_CONTENT_WORDS || tokensB.size() < MIN_CONTENT_WORDS) return 0.0;

        Set<String> intersection = new HashSet<>(tokensA);
        intersection.retainAll(tokensB);
        if (intersection.isEmpty()) return 0.0;

        Set<String> union = new HashSet<>(tokensA);
        union.addAll(tokensB);
        return (double) intersection.size() / union.size();
    }

    public static boolean isDuplicate(String a, String b) {
        return similarity(a, b) >= DUPLICATE_THRESHOLD;
    }

    /** Lowercased, punctuation-stripped, stopword-free word set. */
    static Set<String> contentWords(String remark) {
        Set<String> words = new LinkedHashSet<>();
        if (remark == null) return words;
        for (String raw : remark.toLowerCase().split("[^\\p{L}\\p{N}]+")) {
            if (raw.isEmpty() || STOPWORDS.contains(raw)) continue;
            words.add(raw);
        }
        return words;
    }
}