Subversion Repositories SmartDukaan

package com.smartdukaan.cron.monitored;

import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;

import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Tags;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.stereotype.Component;

/**
 * The per-brand IMEI activation funnel, on /actuator/prometheus.
 *
 * Until now the only way to know whether a brand was working was to grep free text out of
 * a 1.2GB cron.log and reconstruct the funnel with awk. That is how two outages stayed
 * invisible: oppo returned an empty map for a week (same 10 imeis recycling every 5
 * minutes, nothing written), and the vivo captcha solver was dead for 46 days. Both had
 * the same shape and neither raised anything.
 *
 * The funnel is deliberately staged, because WHICH stage stalls says what broke:
 *
 *   due        -> pool the pass snapshotted. 0 with a large backlog = the query is wrong.
 *   churned    -> imeis the pass actually reached. Short of `due` = the pass ran out of
 *                 day. This is the capacity signal.
 *   captcha_shown  -> got as far as a rendered captcha. Collapses when the far end stops
 *                 serving the widget -- realme's rate-limit signature, which looks like
 *                 nothing else in the funnel.
 *   captcha_solved -> we broke it. shown/solved is the solver's real accuracy.
 *   answered   -> the portal gave a verdict. THIS is what bumps createTimestamp and lets
 *                 an imei leave the queue, so answered==0 while churned>0 is the precise
 *                 signature of both historical outages.
 *   dates_found-> an activation date came back. Always a fraction of `answered`: most
 *                 handsets legitimately are not activated yet, so this is a business
 *                 number, not a health one. Do not alert on it alone.
 *
 * Rates are deliberately NOT published. solved/shown and dates/answered are one division
 * in PromQL and storing them would freeze the numerator and denominator apart.
 *
 * -1 means "no pass has finished since this JVM started", so a scrape before the first
 * run is distinguishable from a genuine zero -- same convention as {@link BalanceGauges}.
 */
@Component
public class ImeiActivationGauges {

    private static final Logger LOGGER = LogManager.getLogger(ImeiActivationGauges.class);

    private static final int NOT_RUN_YET = -1;

    private final MeterRegistry meterRegistry;
    private final Map<String, BrandFunnel> byBrand = new ConcurrentHashMap<>();

    public ImeiActivationGauges(MeterRegistry meterRegistry) {
        this.meterRegistry = meterRegistry;
    }

    /** Called once at the start of a brand's daily pass, with the snapshotted pool size. */
    public void beginPass(String brand, int due) {
        BrandFunnel f = funnel(brand);
        f.reset(due);
        LOGGER.info("[{}] pass begin: {} imeis due", brand, due);
    }

    /** A captcha widget rendered and is ready to be attacked. */
    public void captchaShown(String brand) {
        funnel(brand).captchaShown.incrementAndGet();
    }

    /** A captcha was broken. */
    public void captchaSolved(String brand) {
        funnel(brand).captchaSolved.incrementAndGet();
    }

    /**
     * The portal returned a verdict for one imei. {@code dateFound} is false for a
     * legitimate "not activated yet" -- that is still an answer, and still lets the row
     * leave the queue.
     */
    public void answered(String brand, boolean dateFound) {
        BrandFunnel f = funnel(brand);
        f.answered.incrementAndGet();
        if (dateFound) {
            f.datesFound.incrementAndGet();
        }
    }

    /** One imei was consumed from the pass, whatever the outcome. */
    public void churned(String brand, int count) {
        funnel(brand).churned.addAndGet(count);
    }

    /** An exception escaped somewhere in this brand's pass. */
    public void error(String brand) {
        funnel(brand).errors.incrementAndGet();
    }

    /**
     * Called once when a brand's pass ends. Stamps the duration and the completion time,
     * and logs the one line worth grepping for.
     */
    public void endPass(String brand) {
        BrandFunnel f = funnel(brand);
        long seconds = (System.currentTimeMillis() - f.startedAtMillis) / 1000;
        f.runSeconds.set(seconds);
        f.lastFinishEpoch.set(System.currentTimeMillis() / 1000);

        int shown = f.captchaShown.get();
        int solved = f.captchaSolved.get();
        int answered = f.answered.get();
        LOGGER.info("[{}] pass end: due={} churned={} captcha {}/{} ({}%) answered={} dates={} errors={} in {}s",
                brand, f.due.get(), f.churned.get(), solved, shown, percent(solved, shown),
                answered, f.datesFound.get(), f.errors.get(), seconds);

        if (f.churned.get() > 0 && answered == 0) {
            LOGGER.error("[{}] pass answered NOTHING across {} imeis -- the brand is down, not merely unlucky",
                    brand, f.churned.get());
        }
        if (f.churned.get() < f.due.get()) {
            LOGGER.warn("[{}] pass did not finish: {} of {} imeis churned, the tail rolls into tomorrow",
                    brand, f.churned.get(), f.due.get());
        }
    }

    private static int percent(int part, int whole) {
        return whole == 0 ? 0 : (int) Math.round(100.0 * part / whole);
    }

    private BrandFunnel funnel(String brand) {
        return byBrand.computeIfAbsent(brand, b -> new BrandFunnel(meterRegistry, b));
    }

    /**
     * One brand's counters, registered against the meter registry the first time that
     * brand is seen. Gauges hold last-pass values rather than lifetime totals: these
     * passes run once a day, so "what did the last run do" is the question being asked,
     * and a counter that only moves once a day is awkward to alert on.
     */
    private static final class BrandFunnel {

        private final AtomicInteger due = new AtomicInteger(NOT_RUN_YET);
        private final AtomicInteger churned = new AtomicInteger(NOT_RUN_YET);
        private final AtomicInteger captchaShown = new AtomicInteger(NOT_RUN_YET);
        private final AtomicInteger captchaSolved = new AtomicInteger(NOT_RUN_YET);
        private final AtomicInteger answered = new AtomicInteger(NOT_RUN_YET);
        private final AtomicInteger datesFound = new AtomicInteger(NOT_RUN_YET);
        private final AtomicInteger errors = new AtomicInteger(NOT_RUN_YET);
        private final AtomicLong runSeconds = new AtomicLong(NOT_RUN_YET);
        private final AtomicLong lastFinishEpoch = new AtomicLong(NOT_RUN_YET);

        private volatile long startedAtMillis = System.currentTimeMillis();

        BrandFunnel(MeterRegistry registry, String brand) {
            Tags tags = Tags.of("brand", brand.toLowerCase());
            gauge(registry, "imei_activation_due", tags, due,
                    "Imeis the last pass snapshotted for this brand; -1 = no pass yet");
            gauge(registry, "imei_activation_churned", tags, churned,
                    "Imeis the last pass actually reached; short of due means it ran out of day");
            gauge(registry, "imei_activation_captcha_shown", tags, captchaShown,
                    "Captchas that rendered; collapses when the far end stops serving the widget");
            gauge(registry, "imei_activation_captcha_solved", tags, captchaSolved,
                    "Captchas broken; divide by captcha_shown for the solver's accuracy");
            gauge(registry, "imei_activation_answered", tags, answered,
                    "Imeis the portal gave a verdict for; this is what lets a row leave the queue");
            gauge(registry, "imei_activation_dates_found", tags, datesFound,
                    "Activation dates obtained; a fraction of answered by nature, not a health signal");
            gauge(registry, "imei_activation_errors", tags, errors,
                    "Exceptions during the last pass for this brand");
            Gauge.builder("imei_activation_run_seconds", runSeconds, AtomicLong::get)
                    .tags(tags)
                    .description("Wall-clock seconds of the last pass; -1 = no pass yet")
                    .register(registry);
            Gauge.builder("imei_activation_last_finish_epoch_seconds", lastFinishEpoch, AtomicLong::get)
                    .tags(tags)
                    .description("Unix time the last pass finished; staleness alerts key off this")
                    .register(registry);
        }

        private static void gauge(MeterRegistry registry, String name, Tags tags,
                                  AtomicInteger value, String description) {
            Gauge.builder(name, value, AtomicInteger::get)
                    .tags(tags)
                    .description(description)
                    .register(registry);
        }

        void reset(int dueCount) {
            startedAtMillis = System.currentTimeMillis();
            due.set(dueCount);
            churned.set(0);
            captchaShown.set(0);
            captchaSolved.set(0);
            answered.set(0);
            datesFound.set(0);
            errors.set(0);
        }
    }
}