Subversion Repositories SmartDukaan

package com.smartdukaan.cron.monitored;

import java.time.LocalDate;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;

import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Tags;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.stereotype.Component;

/**
 * The per-brand IMEI activation funnel, on /actuator/prometheus.
 *
 * Until now the only way to know whether a brand was working was to grep free text out of
 * a 1.2GB cron.log and reconstruct the funnel with awk. That is how two outages stayed
 * invisible: oppo returned an empty map for a week (same 10 imeis recycling every 5
 * minutes, nothing written), and the vivo captcha solver was dead for 46 days. Both had
 * the same shape and neither raised anything.
 *
 * The funnel is deliberately staged, because WHICH stage stalls says what broke:
 *
 *   due        -> pool the pass snapshotted. 0 with a large backlog = the query is wrong.
 *   churned    -> imeis the pass actually reached. Short of `due` = the pass ran out of
 *                 day. This is the capacity signal.
 *   captcha_shown  -> got as far as a rendered captcha. Collapses when the far end stops
 *                 serving the widget -- realme's rate-limit signature, which looks like
 *                 nothing else in the funnel.
 *   captcha_solved -> we broke it. shown/solved is the solver's real accuracy.
 *   answered   -> the portal gave a verdict. THIS is what bumps createTimestamp and lets
 *                 an imei leave the queue, so answered==0 while churned>0 is the precise
 *                 signature of both historical outages.
 *   dates_found-> an activation date came back. Always a fraction of `answered`: most
 *                 handsets legitimately are not activated yet, so this is a business
 *                 number, not a health one. Do not alert on it alone.
 *
 * Rates are deliberately NOT published. solved/shown and dates/answered are one division
 * in PromQL and storing them would freeze the numerator and denominator apart.
 *
 * -1 means "no day has been cleared since this JVM started", so a scrape before the first
 * tick is distinguishable from a genuine zero -- same convention as {@link BalanceGauges}.
 *
 * ⚠ The gauges are registered lazily, on the first tick that names a brand. After a
 * restart the whole imei_activation_* family is therefore ABSENT from /actuator/prometheus
 * rather than reading -1, until each lane takes its first turn. Under the old midnight
 * cron that blind window could last most of a day; under a 20-second tick it is seconds.
 */
@Component
public class ImeiActivationGauges {

    private static final Logger LOGGER = LogManager.getLogger(ImeiActivationGauges.class);

    private static final int NOT_RUN_YET = -1;

    private final MeterRegistry meterRegistry;
    private final Map<String, BrandFunnel> byBrand = new ConcurrentHashMap<>();

    public ImeiActivationGauges(MeterRegistry meterRegistry) {
        this.meterRegistry = meterRegistry;
    }

    /**
     * True while this brand has not yet opened today's funnel.
     *
     * The caller uses it to decide how much pool to fetch: on the day's first tick it
     * needs the whole thing to have a denominator, on every later tick it needs only the
     * next chunk. Asking here keeps that decision in one place rather than duplicating a
     * date field per lane.
     */
    public boolean needsDayStart(String brand) {
        return !LocalDate.now().equals(funnel(brand).day);
    }

    /**
     * Open a brand's funnel for today, with the pool size measured on the first tick.
     *
     * The unit used to be a pass -- one bounded run that began and ended inside a single
     * call. Ticks have no such boundary, so the DAY is the unit now: `due` is what was
     * outstanding at the first tick after midnight, and everything else accumulates
     * across the day's ticks until the pool comes back empty.
     *
     * Truncation is detected HERE rather than at the end of a run, because a day that
     * ran out of hours never reaches endDay at all -- it just stops. Rolling over with
     * yesterday still open is exactly that case, and it is the line worth alerting on.
     */
    public void beginDay(String brand, int due) {
        BrandFunnel f = funnel(brand);
        if (f.day != null && !f.finished) {
            LOGGER.warn("[{}] {} ended without clearing: {} of {} imeis churned, the tail rolls into today",
                    brand, f.day, f.churned.get(), f.due.get());
        }
        f.reset(due);
        f.day = LocalDate.now();
        f.finished = false;
        LOGGER.info("[{}] day begin: {} imeis due", brand, due);
    }

    /** A captcha widget rendered and is ready to be attacked. */
    public void captchaShown(String brand) {
        funnel(brand).captchaShown.incrementAndGet();
    }

    /** A captcha was broken. */
    public void captchaSolved(String brand) {
        funnel(brand).captchaSolved.incrementAndGet();
    }

    /**
     * The portal returned a verdict for one imei. {@code dateFound} is false for a
     * legitimate "not activated yet" -- that is still an answer, and still lets the row
     * leave the queue.
     */
    public void answered(String brand, boolean dateFound) {
        BrandFunnel f = funnel(brand);
        f.answered.incrementAndGet();
        if (dateFound) {
            f.datesFound.incrementAndGet();
        }
    }

    /** One imei was consumed from the pass, whatever the outcome. */
    public void churned(String brand, int count) {
        funnel(brand).churned.addAndGet(count);
    }

    /** An exception escaped somewhere in this brand's pass. */
    public void error(String brand) {
        funnel(brand).errors.incrementAndGet();
    }

    /**
     * Called the first time a brand's turn comes back with nothing due, which under a
     * tick model is what "the day's work is cleared" means. Stamps the duration and the
     * completion time, and logs the one line worth grepping for.
     *
     * Idempotent by design: every later tick that day also finds an empty pool and calls
     * this, and must stay silent. Silence is the point -- at a 20-second cadence a log
     * line per idle tick is 4,320 lines a day per brand on a log that is already 1.8GB.
     *
     * lastFinishEpoch therefore becomes a genuine completion clock. It read -1 for three
     * days straight under the old model, which said only "no pass has finished since this
     * JVM started" and could not distinguish a truncated pass from a fresh restart.
     */
    public void endDay(String brand) {
        BrandFunnel f = funnel(brand);
        if (!LocalDate.now().equals(f.day) || f.finished) {
            return;
        }
        f.finished = true;
        long seconds = (System.currentTimeMillis() - f.startedAtMillis) / 1000;
        f.runSeconds.set(seconds);
        f.lastFinishEpoch.set(System.currentTimeMillis() / 1000);

        int shown = f.captchaShown.get();
        int solved = f.captchaSolved.get();
        int answered = f.answered.get();
        LOGGER.info("[{}] day cleared: due={} churned={} captcha {}/{} ({}%) answered={} dates={} errors={} in {}s",
                brand, f.due.get(), f.churned.get(), solved, shown, percent(solved, shown),
                answered, f.datesFound.get(), f.errors.get(), seconds);

        if (f.churned.get() > 0 && answered == 0) {
            LOGGER.error("[{}] answered NOTHING across {} imeis -- the brand is down, not merely unlucky",
                    brand, f.churned.get());
        }
        // churned short of due is normal now rather than a truncation: the pool is
        // re-queried every tick, so anything another writer filled in mid-day (carlcare
        // and the DCR pull both write rows) leaves without this lane ever touching it.
        // Real truncation is the day that never gets here at all, caught in beginDay.
    }

    private static int percent(int part, int whole) {
        return whole == 0 ? 0 : (int) Math.round(100.0 * part / whole);
    }

    private BrandFunnel funnel(String brand) {
        return byBrand.computeIfAbsent(brand, b -> new BrandFunnel(meterRegistry, b));
    }

    /**
     * One brand's counters, registered against the meter registry the first time that
     * brand is seen. Gauges hold the CURRENT DAY's values rather than lifetime totals:
     * the question being asked is "how far has today got", and a lifetime counter cannot
     * answer it. They climb across the day's ticks and reset at the first tick after
     * midnight.
     */
    private static final class BrandFunnel {

        private final AtomicInteger due = new AtomicInteger(NOT_RUN_YET);
        private final AtomicInteger churned = new AtomicInteger(NOT_RUN_YET);
        private final AtomicInteger captchaShown = new AtomicInteger(NOT_RUN_YET);
        private final AtomicInteger captchaSolved = new AtomicInteger(NOT_RUN_YET);
        private final AtomicInteger answered = new AtomicInteger(NOT_RUN_YET);
        private final AtomicInteger datesFound = new AtomicInteger(NOT_RUN_YET);
        private final AtomicInteger errors = new AtomicInteger(NOT_RUN_YET);
        private final AtomicLong runSeconds = new AtomicLong(NOT_RUN_YET);
        private final AtomicLong lastFinishEpoch = new AtomicLong(NOT_RUN_YET);

        private volatile long startedAtMillis = System.currentTimeMillis();

        /** The day this funnel is currently accumulating; null until the first tick ever. */
        private volatile LocalDate day;

        /** Whether that day's pool has already come back empty. Keeps endDay silent after the first. */
        private volatile boolean finished;

        BrandFunnel(MeterRegistry registry, String brand) {
            Tags tags = Tags.of("brand", brand.toLowerCase());
            gauge(registry, "imei_activation_due", tags, due,
                    "Imeis outstanding at the first tick after midnight; -1 = no tick yet");
            gauge(registry, "imei_activation_churned", tags, churned,
                    "Imeis today's ticks have reached so far; catches up to due when the day clears");
            gauge(registry, "imei_activation_captcha_shown", tags, captchaShown,
                    "Captchas that rendered; collapses when the far end stops serving the widget");
            gauge(registry, "imei_activation_captcha_solved", tags, captchaSolved,
                    "Captchas broken; divide by captcha_shown for the solver's accuracy");
            gauge(registry, "imei_activation_answered", tags, answered,
                    "Imeis the portal gave a verdict for; this is what lets a row leave the queue");
            gauge(registry, "imei_activation_dates_found", tags, datesFound,
                    "Activation dates obtained; a fraction of answered by nature, not a health signal");
            gauge(registry, "imei_activation_errors", tags, errors,
                    "Exceptions during the last pass for this brand");
            Gauge.builder("imei_activation_run_seconds", runSeconds, AtomicLong::get)
                    .tags(tags)
                    .description("Wall-clock seconds of the last pass; -1 = no pass yet")
                    .register(registry);
            Gauge.builder("imei_activation_last_finish_epoch_seconds", lastFinishEpoch, AtomicLong::get)
                    .tags(tags)
                    .description("Unix time the last pass finished; staleness alerts key off this")
                    .register(registry);
        }

        private static void gauge(MeterRegistry registry, String name, Tags tags,
                                  AtomicInteger value, String description) {
            Gauge.builder(name, value, AtomicInteger::get)
                    .tags(tags)
                    .description(description)
                    .register(registry);
        }

        void reset(int dueCount) {
            startedAtMillis = System.currentTimeMillis();
            due.set(dueCount);
            churned.set(0);
            captchaShown.set(0);
            captchaSolved.set(0);
            answered.set(0);
            datesFound.set(0);
            errors.set(0);
        }
    }
}