Subversion Repositories SmartDukaan

package com.smartdukaan.cron.monitored;

import java.util.Map;

import java.util.concurrent.ConcurrentHashMap;

import java.util.concurrent.atomic.AtomicInteger;

import java.util.concurrent.atomic.AtomicLong;

import io.micrometer.core.instrument.Gauge;

import io.micrometer.core.instrument.MeterRegistry;

import io.micrometer.core.instrument.Tags;

import org.apache.logging.log4j.LogManager;

import org.apache.logging.log4j.Logger;

import org.springframework.stereotype.Component;

/**

 * The per-brand IMEI activation funnel, on /actuator/prometheus.

 *

 * Until now the only way to know whether a brand was working was to grep free text out of

 * a 1.2GB cron.log and reconstruct the funnel with awk. That is how two outages stayed

 * invisible: oppo returned an empty map for a week (same 10 imeis recycling every 5

 * minutes, nothing written), and the vivo captcha solver was dead for 46 days. Both had

 * the same shape and neither raised anything.

 *

 * The funnel is deliberately staged, because WHICH stage stalls says what broke:

 *

 *   due        -> pool the pass snapshotted. 0 with a large backlog = the query is wrong.

 *   churned    -> imeis the pass actually reached. Short of `due` = the pass ran out of

 *                 day. This is the capacity signal.

 *   captcha_shown  -> got as far as a rendered captcha. Collapses when the far end stops

 *                 serving the widget -- realme's rate-limit signature, which looks like

 *                 nothing else in the funnel.

 *   captcha_solved -> we broke it. shown/solved is the solver's real accuracy.

 *   answered   -> the portal gave a verdict. THIS is what bumps createTimestamp and lets

 *                 an imei leave the queue, so answered==0 while churned>0 is the precise

 *                 signature of both historical outages.

 *   dates_found-> an activation date came back. Always a fraction of `answered`: most

 *                 handsets legitimately are not activated yet, so this is a business

 *                 number, not a health one. Do not alert on it alone.

 *

 * Rates are deliberately NOT published. solved/shown and dates/answered are one division

 * in PromQL and storing them would freeze the numerator and denominator apart.

 *

 * -1 means "no pass has finished since this JVM started", so a scrape before the first

 * run is distinguishable from a genuine zero -- same convention as {@link BalanceGauges}.

 */

@Component

public class ImeiActivationGauges {

    private static final Logger LOGGER = LogManager.getLogger(ImeiActivationGauges.class);

    private static final int NOT_RUN_YET = -1;

    private final MeterRegistry meterRegistry;

    private final Map<String, BrandFunnel> byBrand = new ConcurrentHashMap<>();

    public ImeiActivationGauges(MeterRegistry meterRegistry) {

        this.meterRegistry = meterRegistry;

    }

    /** Called once at the start of a brand's daily pass, with the snapshotted pool size. */

    public void beginPass(String brand, int due) {

        BrandFunnel f = funnel(brand);

        f.reset(due);

        LOGGER.info("[{}] pass begin: {} imeis due", brand, due);

    }

    /** A captcha widget rendered and is ready to be attacked. */

    public void captchaShown(String brand) {

        funnel(brand).captchaShown.incrementAndGet();

    }

    /** A captcha was broken. */

    public void captchaSolved(String brand) {

        funnel(brand).captchaSolved.incrementAndGet();

    }

    /**

     * The portal returned a verdict for one imei. {@code dateFound} is false for a

     * legitimate "not activated yet" -- that is still an answer, and still lets the row

     * leave the queue.

     */

    public void answered(String brand, boolean dateFound) {

        BrandFunnel f = funnel(brand);

        f.answered.incrementAndGet();

        if (dateFound) {

            f.datesFound.incrementAndGet();

        }

    }

    /** One imei was consumed from the pass, whatever the outcome. */

    public void churned(String brand, int count) {

        funnel(brand).churned.addAndGet(count);

    }

    /** An exception escaped somewhere in this brand's pass. */

    public void error(String brand) {

        funnel(brand).errors.incrementAndGet();

    }

    /**

     * Called once when a brand's pass ends. Stamps the duration and the completion time,

     * and logs the one line worth grepping for.

     */

    public void endPass(String brand) {

        BrandFunnel f = funnel(brand);

        long seconds = (System.currentTimeMillis() - f.startedAtMillis) / 1000;

        f.runSeconds.set(seconds);

        f.lastFinishEpoch.set(System.currentTimeMillis() / 1000);

        int shown = f.captchaShown.get();

        int solved = f.captchaSolved.get();

        int answered = f.answered.get();

        LOGGER.info("[{}] pass end: due={} churned={} captcha {}/{} ({}%) answered={} dates={} errors={} in {}s",

                brand, f.due.get(), f.churned.get(), solved, shown, percent(solved, shown),

                answered, f.datesFound.get(), f.errors.get(), seconds);

        if (f.churned.get() > 0 && answered == 0) {

            LOGGER.error("[{}] pass answered NOTHING across {} imeis -- the brand is down, not merely unlucky",

                    brand, f.churned.get());

        }

        if (f.churned.get() < f.due.get()) {

            LOGGER.warn("[{}] pass did not finish: {} of {} imeis churned, the tail rolls into tomorrow",

                    brand, f.churned.get(), f.due.get());

        }

    }

    private static int percent(int part, int whole) {

        return whole == 0 ? 0 : (int) Math.round(100.0 * part / whole);

    }

    private BrandFunnel funnel(String brand) {

        return byBrand.computeIfAbsent(brand, b -> new BrandFunnel(meterRegistry, b));

    }

    /**

     * One brand's counters, registered against the meter registry the first time that

     * brand is seen. Gauges hold last-pass values rather than lifetime totals: these

     * passes run once a day, so "what did the last run do" is the question being asked,

     * and a counter that only moves once a day is awkward to alert on.

     */

    private static final class BrandFunnel {

        private final AtomicInteger due = new AtomicInteger(NOT_RUN_YET);

        private final AtomicInteger churned = new AtomicInteger(NOT_RUN_YET);

        private final AtomicInteger captchaShown = new AtomicInteger(NOT_RUN_YET);

        private final AtomicInteger captchaSolved = new AtomicInteger(NOT_RUN_YET);

        private final AtomicInteger answered = new AtomicInteger(NOT_RUN_YET);

        private final AtomicInteger datesFound = new AtomicInteger(NOT_RUN_YET);

        private final AtomicInteger errors = new AtomicInteger(NOT_RUN_YET);

        private final AtomicLong runSeconds = new AtomicLong(NOT_RUN_YET);

        private final AtomicLong lastFinishEpoch = new AtomicLong(NOT_RUN_YET);

        private volatile long startedAtMillis = System.currentTimeMillis();

        BrandFunnel(MeterRegistry registry, String brand) {

            Tags tags = Tags.of("brand", brand.toLowerCase());

            gauge(registry, "imei_activation_due", tags, due,

                    "Imeis the last pass snapshotted for this brand; -1 = no pass yet");

            gauge(registry, "imei_activation_churned", tags, churned,

                    "Imeis the last pass actually reached; short of due means it ran out of day");

            gauge(registry, "imei_activation_captcha_shown", tags, captchaShown,

                    "Captchas that rendered; collapses when the far end stops serving the widget");

            gauge(registry, "imei_activation_captcha_solved", tags, captchaSolved,

                    "Captchas broken; divide by captcha_shown for the solver's accuracy");

            gauge(registry, "imei_activation_answered", tags, answered,

                    "Imeis the portal gave a verdict for; this is what lets a row leave the queue");

            gauge(registry, "imei_activation_dates_found", tags, datesFound,

                    "Activation dates obtained; a fraction of answered by nature, not a health signal");

            gauge(registry, "imei_activation_errors", tags, errors,

                    "Exceptions during the last pass for this brand");

            Gauge.builder("imei_activation_run_seconds", runSeconds, AtomicLong::get)

                    .tags(tags)

                    .description("Wall-clock seconds of the last pass; -1 = no pass yet")

                    .register(registry);

            Gauge.builder("imei_activation_last_finish_epoch_seconds", lastFinishEpoch, AtomicLong::get)

                    .tags(tags)

                    .description("Unix time the last pass finished; staleness alerts key off this")

                    .register(registry);

        }

        private static void gauge(MeterRegistry registry, String name, Tags tags,

                                  AtomicInteger value, String description) {

            Gauge.builder(name, value, AtomicInteger::get)

                    .tags(tags)

                    .description(description)

                    .register(registry);

        }

        void reset(int dueCount) {

            startedAtMillis = System.currentTimeMillis();

            due.set(dueCount);

            churned.set(0);

            captchaShown.set(0);

            captchaSolved.set(0);

            answered.set(0);

            datesFound.set(0);

            errors.set(0);

        }

    }

}