Subversion Repositories SmartDukaan

Rev

Rev 37472 | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 37472 Rev 37565
Line 51... Line 51...
51 
 
 51 
 
52 
    /** Safety stop so a mis-set DAYS cannot pull an unbounded list into memory. */
 52 
    /** Safety stop so a mis-set DAYS cannot pull an unbounded list into memory. */
53 
    private static final int POOL_CAP = 20000;
 53 
    private static final int POOL_CAP = 20000;
54 
 
 54 
 
55 
    /**
 55 
    /**
- 
 
 56 
     * Imeis per tick. Ten times the browser lane's because this is direct HTTP, not a
- 
 
 57 
     * ChromeDriver: vivo measured 9,723 imeis in 39 minutes on 09-Sep, 0.24s each against
- 
 
 58 
     * oppo's 10.2s. At 250 a tick that is ~60s of work per turn and ~52 minutes for the
- 
 
 59 
     * whole pool -- a chunk small enough that a restart costs little, large enough that
- 
 
 60 
     * the 20s gap between ticks stays a rounding error rather than a third of the budget.
- 
 
 61 
     */
- 
 
 62 
    private static final int CHUNK = 250;
- 
 
 63 
 
- 
 
 64 
    /**
56 
     * ONE FULL PASS PER DAY over everything due, then nothing until tomorrow.
 65 
     * The day this lane is working through; null until the first tick ever.
57 
     *
 66 
     *
- 
 
 67 
     * volatile because consecutive ticks of a fixedDelay job are serialised but NOT
- 
 
 68 
     * pinned to one thread -- they land on whichever scheduled-task-pool thread is free.
- 
 
 69 
     * The executor does establish happens-before between them, so this is belt and
- 
 
 70 
     * braces; without it a stale read would re-log the day boundary and re-fetch the
- 
 
 71 
     * whole pool on every tick.
- 
 
 72 
     */
- 
 
 73 
    private volatile LocalDate day;
- 
 
 74 
 
- 
 
 75 
    /** Whether today's pool has already come back empty, so the cleared line is logged once. */
- 
 
 76 
    private volatile boolean cleared;
- 
 
 77 
 
- 
 
 78 
    /**
- 
 
 79 
     * ONE CHUNK PER TICK until the day's pool is empty, then nothing until tomorrow.
- 
 
 80 
     *
- 
 
 81 
     * Called every 20 seconds on its OWN lane. Vivo must not share the browser lane: it
- 
 
 82 
     * is direct HTTP plus the captcha solver at 0.24s an imei, against oppo's 10.2s and
- 
 
 83 
     * realme's 14.2s through a ChromeDriver. Behind them in one rotation its ~9,700 imeis
- 
 
 84 
     * would need 389 turns at roughly eleven minutes a cycle -- about 70 hours, for work
- 
 
 85 
     * that takes it 39 minutes alone. Separate lanes also mean it never contends for the
- 
 
 86 
     * resource the browser lane is rationing, since it opens no driver at all.
- 
 
 87 
     *
58 
     * This was two @Scheduled jobs running in parallel every 5 minutes, capped at 50
 88 
     * Before the daily pass this was two @Scheduled jobs every 5 minutes capped at 50 and
- 
 
 89 
     * 10 imeis, which never finished a pass over a 9,510-row pool and retried a failed
59 
     * (secondary) and 10 (tertiary) imeis a tick -- so it never finished a pass over a
 90 
     * lookup five minutes later instead of the next day. Returning to a tick does NOT
60 
     * 9,510-row pool, and any imei whose lookup failed was retried five minutes later
 91 
     * reopen that. The runaway was caused by failures writing no row, and every outcome
- 
 
 92 
     * vivo actually ANSWERS is stamped: status 2, an empty dueTime and a real date all
61 
     * rather than the next day. The pool is snapshotted here and walked to the end; it
 93 
     * reach saveActivation, so those imeis leave today's pool. The pool query is the only
62 
     * is never re-queried mid-pass, so no imei is asked twice inside a run.
 94 
     * cursor; there is no snapshot to lose to a restart.
63 
     *
 95 
     *
- 
 
 96 
     * Three paths still stamp nothing, and none of them can run away:
- 
 
 97 
     *   - a REJECTED CAPTCHA returns without a row, so the imei comes back on a later
64 
     * The secondary/tertiary split is only two join paths to the same serial, so they
 98 
     *     tick. That converges rather than loops -- the solver is right about 89% of the
65 
     * are concatenated into one list. distinct() is insurance: the secondary query
 99 
     *     time, so an imei needs ~1.1 attempts -- and it is strictly better than the daily
- 
 
 100 
     *     pass, where one wrong captcha cost that imei a full day.
- 
 
 101 
     *   - a line item with NO SERIAL is skipped, and the pool query excludes it anyway.
- 
 
 102 
     *   - a DEAD SOLVER abandons the tick, not the day. It spends one request per 20s
66 
     * already excludes anything carrying a FofoLineItem.
 103 
     *     instead of a chunk's worth, and recovers by itself.
67 
     *
 104 
     *
68 
     * The 1st-of-month skip is kept -- vivo's own portal is unreliable on that day.
 105 
     * The 1st-of-month skip is kept -- vivo's own portal is unreliable on that day.
69 
     */
 106 
     */
70 
    @Transactional(propagation = Propagation.NOT_SUPPORTED)
 107 
    @Transactional(propagation = Propagation.NOT_SUPPORTED)
71 
    public void checkImeiActivation() throws Exception {
 108 
    public void checkImeiActivation() throws Exception {
72 
        if (LocalDate.now().getDayOfMonth() == 1) return;
 109 
        if (LocalDate.now().getDayOfMonth() == 1) return;
73 
 
 110 
 
74 
        List<ImeiActivationTimestampModel> pool = new ArrayList<>();
 111 
        boolean dayStart = !LocalDate.now().equals(day);
75 
        pool.addAll(activatedImeiRepository.selectImeiActivationPendingByBrand("Vivo", DAYS, POOL_CAP));
 112 
        List<ImeiActivationTimestampModel> due = pending(dayStart ? POOL_CAP : CHUNK);
76 
        pool.addAll(activatedImeiRepository.selectImeiActivationPendingByBrandTertiary("Vivo", DAYS, POOL_CAP));
 - 
 
77 
 
 113 
 
- 
 
 114 
        if (dayStart) {
- 
 
 115 
            day = LocalDate.now();
- 
 
 116 
            cleared = false;
- 
 
 117 
            LOGGER.info("Vivo day begin: {} imeis due", due.size());
- 
 
 118 
        }
- 
 
 119 
 
- 
 
 120 
        if (due.isEmpty()) {
78 
        List<ImeiActivationTimestampModel> imeisActivationList = pool.stream()
 121 
            // Cleared for today. One line, then silence until midnight -- at a 20-second
- 
 
 122 
            // tick, logging every idle turn is 4,320 lines a day.
79 
                .filter(distinctBySerial())
 123 
            if (!cleared) {
80 
                .collect(Collectors.toList());
 124 
                cleared = true;
81 
        LOGGER.info("Vivo daily pass starting: {} imeis due", imeisActivationList.size());
 125 
                LOGGER.info("Vivo day cleared");
- 
 
 126 
            }
- 
 
 127 
            return;
- 
 
 128 
        }
82 
 
 129 
 
83 
        processImeis(imeisActivationList);
 130 
        processImeis(due.size() <= CHUNK ? due : due.subList(0, CHUNK));
- 
 
 131 
    }
84 
 
 132 
 
- 
 
 133 
    /**
- 
 
 134 
     * Vivo's own pool, as one de-duplicated list capped at maxResults.
- 
 
 135 
     *
- 
 
 136 
     * Unlike the browser lane this concatenates rather than interleaves, and can: vivo
- 
 
 137 
     * clears its whole pool in well under an hour, so the tail is always reached and
- 
 
 138 
     * ordering cannot starve the tertiary queue.
- 
 
 139 
     */
- 
 
 140 
    private List<ImeiActivationTimestampModel> pending(int maxResults) {
85 
        LOGGER.info("Vivo daily pass finished: {} imeis", imeisActivationList.size());
 141 
        List<ImeiActivationTimestampModel> pool = new ArrayList<>();
- 
 
 142 
        pool.addAll(activatedImeiRepository.selectImeiActivationPendingByBrand("Vivo", DAYS, maxResults));
- 
 
 143 
        pool.addAll(activatedImeiRepository.selectImeiActivationPendingByBrandTertiary("Vivo", DAYS, maxResults));
- 
 
 144 
        return pool.stream()
- 
 
 145 
                .filter(distinctBySerial())
- 
 
 146 
                .limit(maxResults)
- 
 
 147 
                .collect(Collectors.toList());
86 
    }
 148 
    }
87 
 
 149 
 
88 
    private static Predicate<ImeiActivationTimestampModel> distinctBySerial() {
 150 
    private static Predicate<ImeiActivationTimestampModel> distinctBySerial() {
89 
        Set<String> seen = new HashSet<>();
 151 
        Set<String> seen = new HashSet<>();
90 
        return m -> seen.add(m.getSerialNumber());
 152 
        return m -> seen.add(m.getSerialNumber());
Line 115... Line 177...
115 
 
 177 
 
116 
                Captcha captcha = this.getNewCaptcha(httpClient, cookieStore);
 178 
                Captcha captcha = this.getNewCaptcha(httpClient, cookieStore);
117 
                if (captcha == null) {
 179 
                if (captcha == null) {
118 
                    // Solver returned something that is not a captcha code (it was down
 180 
                    // Solver returned something that is not a captcha code (it was down
119 
                    // for 46 days returning an nginx 502 page, which used to be submitted
 181 
                    // for 46 days returning an nginx 502 page, which used to be submitted
- 
 
 182 
                    // to Vivo verbatim).
- 
 
 183 
                    //
- 
 
 184 
                    // BREAK, not continue: the solver is a shared dependency, so one null
- 
 
 185 
                    // means the next 249 are almost certainly null too. Continuing spends
- 
 
 186 
                    // a whole chunk of HTTP calls on a box with 957MB of RAM to learn the
- 
 
 187 
                    // same thing 250 times. Abandoning the tick costs nothing -- no row is
120 
                    // to Vivo verbatim). Skip; this IMEI is picked up on the next run.
 188 
                    // stamped, so every imei here is still due -- and the next tick probes
- 
 
 189 
                    // again in 20 seconds with a single call. Deliberately NOT stamped:
- 
 
 190 
                    // a solver outage is not a verdict on the imei, and resting the pool
- 
 
 191 
                    // over it would forfeit the day's ~9,700 lookups over a transient blip.
- 
 
 192 
                    LOGGER.warn("Vivo captcha solver returned no code; abandoning this tick");
121 
                    continue;
 193 
                    break;
122 
                }
 194 
                }
123 
                this.getImeiActivation(captcha, httpClient, cookieStore, imeiActivationTimestampModel);
 195 
                this.getImeiActivation(captcha, httpClient, cookieStore, imeiActivationTimestampModel);
124 
            }
 196 
            }
125 
        }
 197 
        }
126 
    }
 198 
    }