Subversion Repositories SmartDukaan

package com.smartdukaan.cron.scheduled;

import com.spice.profitmandi.common.web.client.HttpClientFactory;
import com.spice.profitmandi.dao.model.ImeiActivationTimestampModel;
import com.spice.profitmandi.dao.repository.fofo.ActivatedImeiRepository;
import com.spice.profitmandi.service.inventory.InventoryService;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.json.JSONObject;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;

@Component
class VivoImeiActivationService {

    @Autowired
    InventoryService inventoryService;

    @Autowired
    CaptchaService captchaService;

    @Autowired
    ActivatedImeiRepository activatedImeiRepository;

    private static final Logger LOGGER = LogManager.getLogger(VivoImeiActivationService.class);

    /** Days an imei rests after an ANSWER. 0 = due again in the next day's pass. */
    private static final int DAYS = 0;

    /** Safety stop so a mis-set DAYS cannot pull an unbounded list into memory. */
    private static final int POOL_CAP = 20000;

    /**
     * Imeis per tick. Ten times the browser lane's because this is direct HTTP, not a
     * ChromeDriver: vivo measured 9,723 imeis in 39 minutes on 09-Sep, 0.24s each against
     * oppo's 10.2s. At 250 a tick that is ~60s of work per turn and ~52 minutes for the
     * whole pool -- a chunk small enough that a restart costs little, large enough that
     * the 20s gap between ticks stays a rounding error rather than a third of the budget.
     */
    private static final int CHUNK = 250;

    /**
     * The day this lane is working through; null until the first tick ever.
     *
     * volatile because consecutive ticks of a fixedDelay job are serialised but NOT
     * pinned to one thread -- they land on whichever scheduled-task-pool thread is free.
     * The executor does establish happens-before between them, so this is belt and
     * braces; without it a stale read would re-log the day boundary and re-fetch the
     * whole pool on every tick.
     */
    private volatile LocalDate day;

    /** Whether today's pool has already come back empty, so the cleared line is logged once. */
    private volatile boolean cleared;

    /**
     * ONE CHUNK PER TICK until the day's pool is empty, then nothing until tomorrow.
     *
     * Called every 20 seconds on its OWN lane. Vivo must not share the browser lane: it
     * is direct HTTP plus the captcha solver at 0.24s an imei, against oppo's 10.2s and
     * realme's 14.2s through a ChromeDriver. Behind them in one rotation its ~9,700 imeis
     * would need 389 turns at roughly eleven minutes a cycle -- about 70 hours, for work
     * that takes it 39 minutes alone. Separate lanes also mean it never contends for the
     * resource the browser lane is rationing, since it opens no driver at all.
     *
     * Before the daily pass this was two @Scheduled jobs every 5 minutes capped at 50 and
     * 10 imeis, which never finished a pass over a 9,510-row pool and retried a failed
     * lookup five minutes later instead of the next day. Returning to a tick does NOT
     * reopen that. The runaway was caused by failures writing no row, and every outcome
     * vivo actually ANSWERS is stamped: status 2, an empty dueTime and a real date all
     * reach saveActivation, so those imeis leave today's pool. The pool query is the only
     * cursor; there is no snapshot to lose to a restart.
     *
     * Three paths still stamp nothing, and none of them can run away:
     *   - a REJECTED CAPTCHA returns without a row, so the imei comes back on a later
     *     tick. That converges rather than loops -- the solver is right about 89% of the
     *     time, so an imei needs ~1.1 attempts -- and it is strictly better than the daily
     *     pass, where one wrong captcha cost that imei a full day.
     *   - a line item with NO SERIAL is skipped, and the pool query excludes it anyway.
     *   - a DEAD SOLVER abandons the tick, not the day. It spends one request per 20s
     *     instead of a chunk's worth, and recovers by itself.
     *
     * The 1st-of-month skip is kept -- vivo's own portal is unreliable on that day.
     */
    @Transactional(propagation = Propagation.NOT_SUPPORTED)
    public void checkImeiActivation() throws Exception {
        if (LocalDate.now().getDayOfMonth() == 1) return;

        boolean dayStart = !LocalDate.now().equals(day);
        List<ImeiActivationTimestampModel> due = pending(dayStart ? POOL_CAP : CHUNK);

        if (dayStart) {
            day = LocalDate.now();
            cleared = false;
            LOGGER.info("Vivo day begin: {} imeis due", due.size());
        }

        if (due.isEmpty()) {
            // Cleared for today. One line, then silence until midnight -- at a 20-second
            // tick, logging every idle turn is 4,320 lines a day.
            if (!cleared) {
                cleared = true;
                LOGGER.info("Vivo day cleared");
            }
            return;
        }

        processImeis(due.size() <= CHUNK ? due : due.subList(0, CHUNK));
    }

    /**
     * Vivo's own pool, as one de-duplicated list capped at maxResults.
     *
     * Unlike the browser lane this concatenates rather than interleaves, and can: vivo
     * clears its whole pool in well under an hour, so the tail is always reached and
     * ordering cannot starve the tertiary queue.
     */
    private List<ImeiActivationTimestampModel> pending(int maxResults) {
        List<ImeiActivationTimestampModel> pool = new ArrayList<>();
        pool.addAll(activatedImeiRepository.selectImeiActivationPendingByBrand("Vivo", DAYS, maxResults));
        pool.addAll(activatedImeiRepository.selectImeiActivationPendingByBrandTertiary("Vivo", DAYS, maxResults));
        return pool.stream()
                .filter(distinctBySerial())
                .limit(maxResults)
                .collect(Collectors.toList());
    }

    private static Predicate<ImeiActivationTimestampModel> distinctBySerial() {
        Set<String> seen = new HashSet<>();
        return m -> seen.add(m.getSerialNumber());
    }

    private void processImeis(List<ImeiActivationTimestampModel> imeisActivationList) throws Exception {
        if (imeisActivationList.isEmpty()) return;

        BasicCookieStore cookieStore = new BasicCookieStore();
        try (CloseableHttpClient httpClient = HttpClientFactory.apacheHttpWithCookies(cookieStore)) {
            // Seed the session cookies
            HttpGet seed = new HttpGet("https://www.vivo.com/in/support/IMEI");
            try (CloseableHttpResponse seedResponse = httpClient.execute(seed)) {
                EntityUtils.consumeQuietly(seedResponse.getEntity());
            }

            for (ImeiActivationTimestampModel imeiActivationTimestampModel : imeisActivationList) {
                // Never spend a captcha on a line item with no IMEI. Vivo answers
                // those with {"msg":"参数为空"} ("parameter is empty") and status 0,
                // which this code used to record as a captcha failure -- so a correct
                // code looked wrong, the row was never saved, and it came back every
                // 5 minutes forever. That was 76% of all rejections.
                String serial = imeiActivationTimestampModel.getSerialNumber();
                if (serial == null || serial.trim().isEmpty()) {
                    LOGGER.warn("Skipping activation lookup: line item has no serial number");
                    continue;
                }

                Captcha captcha = this.getNewCaptcha(httpClient, cookieStore);
                if (captcha == null) {
                    // Solver returned something that is not a captcha code (it was down
                    // for 46 days returning an nginx 502 page, which used to be submitted
                    // to Vivo verbatim).
                    //
                    // BREAK, not continue: the solver is a shared dependency, so one null
                    // means the next 249 are almost certainly null too. Continuing spends
                    // a whole chunk of HTTP calls on a box with 957MB of RAM to learn the
                    // same thing 250 times. Abandoning the tick costs nothing -- no row is
                    // stamped, so every imei here is still due -- and the next tick probes
                    // again in 20 seconds with a single call. Deliberately NOT stamped:
                    // a solver outage is not a verdict on the imei, and resting the pool
                    // over it would forfeit the day's ~9,700 lookups over a transient blip.
                    LOGGER.warn("Vivo captcha solver returned no code; abandoning this tick");
                    break;
                }
                this.getImeiActivation(captcha, httpClient, cookieStore, imeiActivationTimestampModel);
            }
        }
    }

    /** A solved captcha: the code, plus the image it came from so the verdict can be reported. */
    static final class Captcha {
        final String code;
        final byte[] image;
        final long fetchedAt;

        Captcha(String code, byte[] image) {
            this.code = code;
            this.image = image;
            this.fetchedAt = System.currentTimeMillis();
        }
    }

    public Captcha getNewCaptcha(CloseableHttpClient httpClient, BasicCookieStore cookieStore) throws Exception {
        HttpGet request = new HttpGet("https://www.vivo.com/in/support/generatingCodes");

        // Kept in memory rather than written to a shared /tmp/captcha.jpg: the two
        // Vivo jobs run concurrently and used to overwrite each other's image. The
        // bytes are needed for the verdict report anyway.
        byte[] bytes;
        try (CloseableHttpResponse response = httpClient.execute(request)) {
            bytes = EntityUtils.toByteArray(response.getEntity());
        }

        String code = captchaService.getCaptchaCode(bytes);
        return code == null ? null : new Captcha(code, bytes);
    }

    public void getImeiActivation(Captcha captcha, CloseableHttpClient httpClient,
                                  BasicCookieStore cookieStore,
                                  ImeiActivationTimestampModel vivoImeiAndActivationTimeStampModel) throws Exception {

        String imei = vivoImeiAndActivationTimeStampModel.getSerialNumber();

        List<NameValuePair> form = new ArrayList<>();
        form.add(new BasicNameValuePair("imei", imei));
        form.add(new BasicNameValuePair("code", captcha.code));
        HttpPost post = new HttpPost("https://www.vivo.com/in/support/checkCode");
        post.setEntity(new UrlEncodedFormEntity(form, StandardCharsets.UTF_8));

        JSONObject imeiActivationJson;
        try (CloseableHttpResponse response2 = httpClient.execute(post)) {
            imeiActivationJson = new JSONObject(EntityUtils.toString(response2.getEntity()));
        }

        JSONObject data = imeiActivationJson.getJSONObject("data");
        int status = data.getInt("status");

        // Vivo is the only judge of whether our guess was right; status 0 means the
        // captcha was wrong. Hand that back so the sample becomes training data.
        captchaService.reportVerdict(captcha.image, captcha.code, status != 0);

        if (status == 0) {
            // Vivo refuses ~56% of submissions in production, but reading the stored
            // images by hand showed 9 of 10 carried the CORRECT code, and no external
            // replication reproduces it (IMEI, user-agent, session reuse, concurrency,
            // source IP and submit delay were all measured and ruled out). The old log
            // line recorded five words and discarded the evidence, so capture it:
            // the code submitted, the imei, how long fetch->submit actually took inside
            // this process, the session cookies held at submit time, and Vivo's reply.
            LOGGER.info("Vivo rejected captcha code={} imei={} elapsedMs={} cookies={} response={}",
                    captcha.code, imei, System.currentTimeMillis() - captcha.fetchedAt,
                    cookieStore.getCookies(), imeiActivationJson);
            return;
        } else if (status == 2) {
            LOGGER.info("Received status 2 for {}, data {}", imei, data);
            activatedImeiRepository.saveActivation(imei, null);
            return;
        }

        JSONObject imeiQueryDto = data.getJSONObject("imeiQueryDto");
        LOGGER.info("imeiQueryDto - {}", imeiQueryDto);
        String dueTimeString = imeiQueryDto.getString("dueTime");

        if (dueTimeString.equals("")) {
            activatedImeiRepository.saveActivation(imei, null);
            return;
        }

        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd/MM/yyyy");
        dueTimeString = dueTimeString.split(" ")[0];
        LocalDate dueDate = LocalDate.parse(dueTimeString, formatter);

        activatedImeiRepository.saveActivation(imei, dueDate.atStartOfDay().minusYears(1));
        LOGGER.info("activatedImei saved for {} with dueDate {}", imei, dueDate.atStartOfDay());
    }
}