| 37391 |
vikas |
1 |
package com.spice.profitmandi.web.services;
|
|
|
2 |
|
|
|
3 |
import java.util.Arrays;
|
|
|
4 |
import java.util.HashSet;
|
|
|
5 |
import java.util.LinkedHashSet;
|
|
|
6 |
import java.util.Set;
|
|
|
7 |
|
|
|
8 |
/**
|
|
|
9 |
* Near-duplicate detection between the remarks of ONE checkout.
|
|
|
10 |
*
|
|
|
11 |
* Why this lives here and not in the analyze service: that service compares a
|
|
|
12 |
* remark only against the ASM's OTHER visits, and matches on normalized-exact
|
|
|
13 |
* text (every hit comes back at similarity 1.0). Both of those miss the case the
|
|
|
14 |
* checkout form actively provokes — the app rejects a partial fill, so a rep with
|
|
|
15 |
* five open agendas and one thing to say pastes it into all five boxes, lightly
|
|
|
16 |
* reworded. Same visit_id, non-identical text: invisible upstream.
|
|
|
17 |
*
|
|
|
18 |
* Content-word overlap (Jaccard) is what we can do without an embedding model.
|
|
|
19 |
* It catches the copy-with-tweaks family — punctuation and filler edits, word
|
|
|
20 |
* reordering, moderate rewording — and it does NOT catch a genuine paraphrase
|
|
|
21 |
* built from different vocabulary. Measured against the same seed remark:
|
|
|
22 |
*
|
|
|
23 |
* "…collected 45,000 … will place the order on Monday." 0.85 flagged
|
|
|
24 |
* "Owner met, collected 45000 towards old dues, order …" 0.64 flagged
|
|
|
25 |
* "Recovered pending payment of 45k … ordering Monday." 0.10 missed
|
|
|
26 |
*
|
|
|
27 |
* Closing that last gap needs sentence embeddings, i.e. an analyze-side change.
|
|
|
28 |
*/
|
|
|
29 |
public final class RemarkSimilarity {
|
|
|
30 |
|
|
|
31 |
private RemarkSimilarity() {}
|
|
|
32 |
|
|
|
33 |
/** Overlap at or above this counts as the same remark reused. */
|
|
|
34 |
public static final double DUPLICATE_THRESHOLD = 0.6;
|
|
|
35 |
|
|
|
36 |
/**
|
|
|
37 |
* Below this many content words a remark is too thin to compare — two such
|
|
|
38 |
* remarks overlap trivially, and the scorer already rejects them outright
|
|
|
39 |
* (TOO_SHORT / SINGLE_WORD).
|
|
|
40 |
*/
|
|
|
41 |
private static final int MIN_CONTENT_WORDS = 3;
|
|
|
42 |
|
|
|
43 |
/** Filler with no bearing on what was discussed; English + common Hinglish. */
|
|
|
44 |
private static final Set<String> STOPWORDS = new HashSet<>(Arrays.asList(
|
|
|
45 |
"a", "an", "the", "and", "or", "but", "also", "as", "at", "by", "for", "from", "in", "into",
|
|
|
46 |
"of", "on", "to", "with", "is", "was", "were", "be", "been", "am", "are", "will", "would",
|
|
|
47 |
"has", "have", "had", "do", "did", "does", "done", "not", "no", "he", "she", "it", "they",
|
|
|
48 |
"we", "i", "his", "her", "their", "our", "my", "them", "him", "that", "this", "these",
|
|
|
49 |
"those", "there", "then", "so", "very", "today", "ka", "ke", "ki", "ko", "kaa", "hai",
|
|
|
50 |
"tha", "the2", "se", "me", "mein", "aur", "nahi", "bhi", "kar", "karke", "kiya", "diya"));
|
|
|
51 |
|
|
|
52 |
/**
|
|
|
53 |
* Jaccard overlap of the two remarks' content words, 0..1. Returns 0 when
|
|
|
54 |
* either side is too thin to judge, so callers can treat any positive result
|
|
|
55 |
* as comparable.
|
|
|
56 |
*/
|
|
|
57 |
public static double similarity(String a, String b) {
|
|
|
58 |
Set<String> tokensA = contentWords(a);
|
|
|
59 |
Set<String> tokensB = contentWords(b);
|
|
|
60 |
if (tokensA.size() < MIN_CONTENT_WORDS || tokensB.size() < MIN_CONTENT_WORDS) return 0.0;
|
|
|
61 |
|
|
|
62 |
Set<String> intersection = new HashSet<>(tokensA);
|
|
|
63 |
intersection.retainAll(tokensB);
|
|
|
64 |
if (intersection.isEmpty()) return 0.0;
|
|
|
65 |
|
|
|
66 |
Set<String> union = new HashSet<>(tokensA);
|
|
|
67 |
union.addAll(tokensB);
|
|
|
68 |
return (double) intersection.size() / union.size();
|
|
|
69 |
}
|
|
|
70 |
|
|
|
71 |
public static boolean isDuplicate(String a, String b) {
|
|
|
72 |
return similarity(a, b) >= DUPLICATE_THRESHOLD;
|
|
|
73 |
}
|
|
|
74 |
|
|
|
75 |
/** Lowercased, punctuation-stripped, stopword-free word set. */
|
|
|
76 |
static Set<String> contentWords(String remark) {
|
|
|
77 |
Set<String> words = new LinkedHashSet<>();
|
|
|
78 |
if (remark == null) return words;
|
|
|
79 |
for (String raw : remark.toLowerCase().split("[^\\p{L}\\p{N}]+")) {
|
|
|
80 |
if (raw.isEmpty() || STOPWORDS.contains(raw)) continue;
|
|
|
81 |
words.add(raw);
|
|
|
82 |
}
|
|
|
83 |
return words;
|
|
|
84 |
}
|
|
|
85 |
}
|