Subversion Repositories SmartDukaan

package com.smartdukaan.cron.scheduled;

import com.spice.profitmandi.common.web.client.HttpClientFactory;

import com.spice.profitmandi.dao.model.ImeiActivationTimestampModel;

import com.spice.profitmandi.dao.repository.fofo.ActivatedImeiRepository;

import com.spice.profitmandi.service.inventory.InventoryService;

import org.apache.http.NameValuePair;

import org.apache.http.client.entity.UrlEncodedFormEntity;

import org.apache.http.client.methods.CloseableHttpResponse;

import org.apache.http.client.methods.HttpGet;

import org.apache.http.client.methods.HttpPost;

import org.apache.http.impl.client.BasicCookieStore;

import org.apache.http.impl.client.CloseableHttpClient;

import org.apache.http.message.BasicNameValuePair;

import org.apache.http.util.EntityUtils;

import org.apache.logging.log4j.LogManager;

import org.apache.logging.log4j.Logger;

import org.json.JSONObject;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.stereotype.Component;

import org.springframework.transaction.annotation.Propagation;

import org.springframework.transaction.annotation.Transactional;

import java.io.IOException;

import java.nio.charset.StandardCharsets;

import java.time.LocalDate;

import java.time.format.DateTimeFormatter;

import java.util.ArrayList;

import java.util.HashSet;

import java.util.List;

import java.util.Set;

import java.util.function.Predicate;

import java.util.stream.Collectors;

@Component

class VivoImeiActivationService {

    @Autowired

    InventoryService inventoryService;

    @Autowired

    CaptchaService captchaService;

    @Autowired

    ActivatedImeiRepository activatedImeiRepository;

    private static final Logger LOGGER = LogManager.getLogger(VivoImeiActivationService.class);

    /** Days an imei rests after an ANSWER. 0 = due again in the next day's pass. */

    private static final int DAYS = 0;

    /** Safety stop so a mis-set DAYS cannot pull an unbounded list into memory. */

    private static final int POOL_CAP = 20000;

    /**

     * Imeis per tick. Ten times the browser lane's because this is direct HTTP, not a

     * ChromeDriver: vivo measured 9,723 imeis in 39 minutes on 09-Sep, 0.24s each against

     * oppo's 10.2s. At 250 a tick that is ~60s of work per turn and ~52 minutes for the

     * whole pool -- a chunk small enough that a restart costs little, large enough that

     * the 20s gap between ticks stays a rounding error rather than a third of the budget.

     */

    private static final int CHUNK = 250;

    /**

     * The day this lane is working through; null until the first tick ever.

     *

     * volatile because consecutive ticks of a fixedDelay job are serialised but NOT

     * pinned to one thread -- they land on whichever scheduled-task-pool thread is free.

     * The executor does establish happens-before between them, so this is belt and

     * braces; without it a stale read would re-log the day boundary and re-fetch the

     * whole pool on every tick.

     */

    private volatile LocalDate day;

    /** Whether today's pool has already come back empty, so the cleared line is logged once. */

    private volatile boolean cleared;

    /**

     * ONE CHUNK PER TICK until the day's pool is empty, then nothing until tomorrow.

     *

     * Called every 20 seconds on its OWN lane. Vivo must not share the browser lane: it

     * is direct HTTP plus the captcha solver at 0.24s an imei, against oppo's 10.2s and

     * realme's 14.2s through a ChromeDriver. Behind them in one rotation its ~9,700 imeis

     * would need 389 turns at roughly eleven minutes a cycle -- about 70 hours, for work

     * that takes it 39 minutes alone. Separate lanes also mean it never contends for the

     * resource the browser lane is rationing, since it opens no driver at all.

     *

     * Before the daily pass this was two @Scheduled jobs every 5 minutes capped at 50 and

     * 10 imeis, which never finished a pass over a 9,510-row pool and retried a failed

     * lookup five minutes later instead of the next day. Returning to a tick does NOT

     * reopen that. The runaway was caused by failures writing no row, and every outcome

     * vivo actually ANSWERS is stamped: status 2, an empty dueTime and a real date all

     * reach saveActivation, so those imeis leave today's pool. The pool query is the only

     * cursor; there is no snapshot to lose to a restart.

     *

     * Three paths still stamp nothing, and none of them can run away:

     *   - a REJECTED CAPTCHA returns without a row, so the imei comes back on a later

     *     tick. That converges rather than loops -- the solver is right about 89% of the

     *     time, so an imei needs ~1.1 attempts -- and it is strictly better than the daily

     *     pass, where one wrong captcha cost that imei a full day.

     *   - a line item with NO SERIAL is skipped, and the pool query excludes it anyway.

     *   - a DEAD SOLVER abandons the tick, not the day. It spends one request per 20s

     *     instead of a chunk's worth, and recovers by itself.

     *

     * The 1st-of-month skip is kept -- vivo's own portal is unreliable on that day.

     */

    @Transactional(propagation = Propagation.NOT_SUPPORTED)

    public void checkImeiActivation() throws Exception {

        if (LocalDate.now().getDayOfMonth() == 1) return;

        boolean dayStart = !LocalDate.now().equals(day);

        List<ImeiActivationTimestampModel> due = pending(dayStart ? POOL_CAP : CHUNK);

        if (dayStart) {

            day = LocalDate.now();

            cleared = false;

            LOGGER.info("Vivo day begin: {} imeis due", due.size());

        }

        if (due.isEmpty()) {

            // Cleared for today. One line, then silence until midnight -- at a 20-second

            // tick, logging every idle turn is 4,320 lines a day.

            if (!cleared) {

                cleared = true;

                LOGGER.info("Vivo day cleared");

            }

            return;

        }

        processImeis(due.size() <= CHUNK ? due : due.subList(0, CHUNK));

    }

    /**

     * Vivo's own pool, as one de-duplicated list capped at maxResults.

     *

     * Unlike the browser lane this concatenates rather than interleaves, and can: vivo

     * clears its whole pool in well under an hour, so the tail is always reached and

     * ordering cannot starve the tertiary queue.

     */

    private List<ImeiActivationTimestampModel> pending(int maxResults) {

        List<ImeiActivationTimestampModel> pool = new ArrayList<>();

        pool.addAll(activatedImeiRepository.selectImeiActivationPendingByBrand("Vivo", DAYS, maxResults));

        pool.addAll(activatedImeiRepository.selectImeiActivationPendingByBrandTertiary("Vivo", DAYS, maxResults));

        return pool.stream()

                .filter(distinctBySerial())

                .limit(maxResults)

                .collect(Collectors.toList());

    }

    private static Predicate<ImeiActivationTimestampModel> distinctBySerial() {

        Set<String> seen = new HashSet<>();

        return m -> seen.add(m.getSerialNumber());

    }

    private void processImeis(List<ImeiActivationTimestampModel> imeisActivationList) throws Exception {

        if (imeisActivationList.isEmpty()) return;

        BasicCookieStore cookieStore = new BasicCookieStore();

        try (CloseableHttpClient httpClient = HttpClientFactory.apacheHttpWithCookies(cookieStore)) {

            // Seed the session cookies

            HttpGet seed = new HttpGet("https://www.vivo.com/in/support/IMEI");

            try (CloseableHttpResponse seedResponse = httpClient.execute(seed)) {

                EntityUtils.consumeQuietly(seedResponse.getEntity());

            }

            for (ImeiActivationTimestampModel imeiActivationTimestampModel : imeisActivationList) {

                // Never spend a captcha on a line item with no IMEI. Vivo answers

                // those with {"msg":"参数为空"} ("parameter is empty") and status 0,

                // which this code used to record as a captcha failure -- so a correct

                // code looked wrong, the row was never saved, and it came back every

                // 5 minutes forever. That was 76% of all rejections.

                String serial = imeiActivationTimestampModel.getSerialNumber();

                if (serial == null || serial.trim().isEmpty()) {

                    LOGGER.warn("Skipping activation lookup: line item has no serial number");

                    continue;

                }

                Captcha captcha = this.getNewCaptcha(httpClient, cookieStore);

                if (captcha == null) {

                    // Solver returned something that is not a captcha code (it was down

                    // for 46 days returning an nginx 502 page, which used to be submitted

                    // to Vivo verbatim).

                    //

                    // BREAK, not continue: the solver is a shared dependency, so one null

                    // means the next 249 are almost certainly null too. Continuing spends

                    // a whole chunk of HTTP calls on a box with 957MB of RAM to learn the

                    // same thing 250 times. Abandoning the tick costs nothing -- no row is

                    // stamped, so every imei here is still due -- and the next tick probes

                    // again in 20 seconds with a single call. Deliberately NOT stamped:

                    // a solver outage is not a verdict on the imei, and resting the pool

                    // over it would forfeit the day's ~9,700 lookups over a transient blip.

                    LOGGER.warn("Vivo captcha solver returned no code; abandoning this tick");

                    break;

                }

                this.getImeiActivation(captcha, httpClient, cookieStore, imeiActivationTimestampModel);

            }

        }

    }

    /** A solved captcha: the code, plus the image it came from so the verdict can be reported. */

    static final class Captcha {

        final String code;

        final byte[] image;

        final long fetchedAt;

        Captcha(String code, byte[] image) {

            this.code = code;

            this.image = image;

            this.fetchedAt = System.currentTimeMillis();

        }

    }

    public Captcha getNewCaptcha(CloseableHttpClient httpClient, BasicCookieStore cookieStore) throws Exception {

        HttpGet request = new HttpGet("https://www.vivo.com/in/support/generatingCodes");

        // Kept in memory rather than written to a shared /tmp/captcha.jpg: the two

        // Vivo jobs run concurrently and used to overwrite each other's image. The

        // bytes are needed for the verdict report anyway.

        byte[] bytes;

        try (CloseableHttpResponse response = httpClient.execute(request)) {

            bytes = EntityUtils.toByteArray(response.getEntity());

        }

        String code = captchaService.getCaptchaCode(bytes);

        return code == null ? null : new Captcha(code, bytes);

    }

    public void getImeiActivation(Captcha captcha, CloseableHttpClient httpClient,

                                  BasicCookieStore cookieStore,

                                  ImeiActivationTimestampModel vivoImeiAndActivationTimeStampModel) throws Exception {

        String imei = vivoImeiAndActivationTimeStampModel.getSerialNumber();

        List<NameValuePair> form = new ArrayList<>();

        form.add(new BasicNameValuePair("imei", imei));

        form.add(new BasicNameValuePair("code", captcha.code));

        HttpPost post = new HttpPost("https://www.vivo.com/in/support/checkCode");

        post.setEntity(new UrlEncodedFormEntity(form, StandardCharsets.UTF_8));

        JSONObject imeiActivationJson;

        try (CloseableHttpResponse response2 = httpClient.execute(post)) {

            imeiActivationJson = new JSONObject(EntityUtils.toString(response2.getEntity()));

        }

        JSONObject data = imeiActivationJson.getJSONObject("data");

        int status = data.getInt("status");

        // Vivo is the only judge of whether our guess was right; status 0 means the

        // captcha was wrong. Hand that back so the sample becomes training data.

        captchaService.reportVerdict(captcha.image, captcha.code, status != 0);

        if (status == 0) {

            // Vivo refuses ~56% of submissions in production, but reading the stored

            // images by hand showed 9 of 10 carried the CORRECT code, and no external

            // replication reproduces it (IMEI, user-agent, session reuse, concurrency,

            // source IP and submit delay were all measured and ruled out). The old log

            // line recorded five words and discarded the evidence, so capture it:

            // the code submitted, the imei, how long fetch->submit actually took inside

            // this process, the session cookies held at submit time, and Vivo's reply.

            LOGGER.info("Vivo rejected captcha code={} imei={} elapsedMs={} cookies={} response={}",

                    captcha.code, imei, System.currentTimeMillis() - captcha.fetchedAt,

                    cookieStore.getCookies(), imeiActivationJson);

            return;

        } else if (status == 2) {

            LOGGER.info("Received status 2 for {}, data {}", imei, data);

            activatedImeiRepository.saveActivation(imei, null);

            return;

        }

        JSONObject imeiQueryDto = data.getJSONObject("imeiQueryDto");

        LOGGER.info("imeiQueryDto - {}", imeiQueryDto);

        String dueTimeString = imeiQueryDto.getString("dueTime");

        if (dueTimeString.equals("")) {

            activatedImeiRepository.saveActivation(imei, null);

            return;

        }

        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd/MM/yyyy");

        dueTimeString = dueTimeString.split(" ")[0];

        LocalDate dueDate = LocalDate.parse(dueTimeString, formatter);

        activatedImeiRepository.saveActivation(imei, dueDate.atStartOfDay().minusYears(1));

        LOGGER.info("activatedImei saved for {} with dueDate {}", imei, dueDate.atStartOfDay());

    }

}