Subversion Repositories SmartDukaan

package com.smartdukaan.cron.monitored;

import java.time.LocalDate;

import java.util.Map;

import java.util.concurrent.ConcurrentHashMap;

import java.util.concurrent.atomic.AtomicInteger;

import java.util.concurrent.atomic.AtomicLong;

import io.micrometer.core.instrument.Gauge;

import io.micrometer.core.instrument.MeterRegistry;

import io.micrometer.core.instrument.Tags;

import org.apache.logging.log4j.LogManager;

import org.apache.logging.log4j.Logger;

import org.springframework.stereotype.Component;

/**

 * The per-brand IMEI activation funnel, on /actuator/prometheus.

 *

 * Until now the only way to know whether a brand was working was to grep free text out of

 * a 1.2GB cron.log and reconstruct the funnel with awk. That is how two outages stayed

 * invisible: oppo returned an empty map for a week (same 10 imeis recycling every 5

 * minutes, nothing written), and the vivo captcha solver was dead for 46 days. Both had

 * the same shape and neither raised anything.

 *

 * The funnel is deliberately staged, because WHICH stage stalls says what broke:

 *

 *   due        -> pool the pass snapshotted. 0 with a large backlog = the query is wrong.

 *   churned    -> imeis the pass actually reached. Short of `due` = the pass ran out of

 *                 day. This is the capacity signal.

 *   captcha_shown  -> got as far as a rendered captcha. Collapses when the far end stops

 *                 serving the widget -- realme's rate-limit signature, which looks like

 *                 nothing else in the funnel.

 *   captcha_solved -> we broke it. shown/solved is the solver's real accuracy.

 *   answered   -> the portal gave a verdict. THIS is what bumps createTimestamp and lets

 *                 an imei leave the queue, so answered==0 while churned>0 is the precise

 *                 signature of both historical outages.

 *   dates_found-> an activation date came back. Always a fraction of `answered`: most

 *                 handsets legitimately are not activated yet, so this is a business

 *                 number, not a health one. Do not alert on it alone.

 *

 * Rates are deliberately NOT published. solved/shown and dates/answered are one division

 * in PromQL and storing them would freeze the numerator and denominator apart.

 *

 * -1 means "no day has been cleared since this JVM started", so a scrape before the first

 * tick is distinguishable from a genuine zero -- same convention as {@link BalanceGauges}.

 *

 * ⚠ The gauges are registered lazily, on the first tick that names a brand. After a

 * restart the whole imei_activation_* family is therefore ABSENT from /actuator/prometheus

 * rather than reading -1, until each lane takes its first turn. Under the old midnight

 * cron that blind window could last most of a day; under a 20-second tick it is seconds.

 */

@Component

public class ImeiActivationGauges {

    private static final Logger LOGGER = LogManager.getLogger(ImeiActivationGauges.class);

    private static final int NOT_RUN_YET = -1;

    private final MeterRegistry meterRegistry;

    private final Map<String, BrandFunnel> byBrand = new ConcurrentHashMap<>();

    public ImeiActivationGauges(MeterRegistry meterRegistry) {

        this.meterRegistry = meterRegistry;

    }

    /**

     * True while this brand has not yet opened today's funnel.

     *

     * The caller uses it to decide how much pool to fetch: on the day's first tick it

     * needs the whole thing to have a denominator, on every later tick it needs only the

     * next chunk. Asking here keeps that decision in one place rather than duplicating a

     * date field per lane.

     */

    public boolean needsDayStart(String brand) {

        return !LocalDate.now().equals(funnel(brand).day);

    }

    /**

     * Open a brand's funnel for today, with the pool size measured on the first tick.

     *

     * The unit used to be a pass -- one bounded run that began and ended inside a single

     * call. Ticks have no such boundary, so the DAY is the unit now: `due` is what was

     * outstanding at the first tick after midnight, and everything else accumulates

     * across the day's ticks until the pool comes back empty.

     *

     * Truncation is detected HERE rather than at the end of a run, because a day that

     * ran out of hours never reaches endDay at all -- it just stops. Rolling over with

     * yesterday still open is exactly that case, and it is the line worth alerting on.

     */

    public void beginDay(String brand, int due) {

        BrandFunnel f = funnel(brand);

        if (f.day != null && !f.finished) {

            LOGGER.warn("[{}] {} ended without clearing: {} of {} imeis churned, the tail rolls into today",

                    brand, f.day, f.churned.get(), f.due.get());

        }

        f.reset(due);

        f.day = LocalDate.now();

        f.finished = false;

        LOGGER.info("[{}] day begin: {} imeis due", brand, due);

    }

    /** A captcha widget rendered and is ready to be attacked. */

    public void captchaShown(String brand) {

        funnel(brand).captchaShown.incrementAndGet();

    }

    /** A captcha was broken. */

    public void captchaSolved(String brand) {

        funnel(brand).captchaSolved.incrementAndGet();

    }

    /**

     * The portal returned a verdict for one imei. {@code dateFound} is false for a

     * legitimate "not activated yet" -- that is still an answer, and still lets the row

     * leave the queue.

     */

    public void answered(String brand, boolean dateFound) {

        BrandFunnel f = funnel(brand);

        f.answered.incrementAndGet();

        if (dateFound) {

            f.datesFound.incrementAndGet();

        }

    }

    /** One imei was consumed from the pass, whatever the outcome. */

    public void churned(String brand, int count) {

        funnel(brand).churned.addAndGet(count);

    }

    /** An exception escaped somewhere in this brand's pass. */

    public void error(String brand) {

        funnel(brand).errors.incrementAndGet();

    }

    /**

     * Called the first time a brand's turn comes back with nothing due, which under a

     * tick model is what "the day's work is cleared" means. Stamps the duration and the

     * completion time, and logs the one line worth grepping for.

     *

     * Idempotent by design: every later tick that day also finds an empty pool and calls

     * this, and must stay silent. Silence is the point -- at a 20-second cadence a log

     * line per idle tick is 4,320 lines a day per brand on a log that is already 1.8GB.

     *

     * lastFinishEpoch therefore becomes a genuine completion clock. It read -1 for three

     * days straight under the old model, which said only "no pass has finished since this

     * JVM started" and could not distinguish a truncated pass from a fresh restart.

     */

    public void endDay(String brand) {

        BrandFunnel f = funnel(brand);

        if (!LocalDate.now().equals(f.day) || f.finished) {

            return;

        }

        f.finished = true;

        long seconds = (System.currentTimeMillis() - f.startedAtMillis) / 1000;

        f.runSeconds.set(seconds);

        f.lastFinishEpoch.set(System.currentTimeMillis() / 1000);

        int shown = f.captchaShown.get();

        int solved = f.captchaSolved.get();

        int answered = f.answered.get();

        LOGGER.info("[{}] day cleared: due={} churned={} captcha {}/{} ({}%) answered={} dates={} errors={} in {}s",

                brand, f.due.get(), f.churned.get(), solved, shown, percent(solved, shown),

                answered, f.datesFound.get(), f.errors.get(), seconds);

        if (f.churned.get() > 0 && answered == 0) {

            LOGGER.error("[{}] answered NOTHING across {} imeis -- the brand is down, not merely unlucky",

                    brand, f.churned.get());

        }

        // churned short of due is normal now rather than a truncation: the pool is

        // re-queried every tick, so anything another writer filled in mid-day (carlcare

        // and the DCR pull both write rows) leaves without this lane ever touching it.

        // Real truncation is the day that never gets here at all, caught in beginDay.

    }

    private static int percent(int part, int whole) {

        return whole == 0 ? 0 : (int) Math.round(100.0 * part / whole);

    }

    private BrandFunnel funnel(String brand) {

        return byBrand.computeIfAbsent(brand, b -> new BrandFunnel(meterRegistry, b));

    }

    /**

     * One brand's counters, registered against the meter registry the first time that

     * brand is seen. Gauges hold the CURRENT DAY's values rather than lifetime totals:

     * the question being asked is "how far has today got", and a lifetime counter cannot

     * answer it. They climb across the day's ticks and reset at the first tick after

     * midnight.

     */

    private static final class BrandFunnel {

        private final AtomicInteger due = new AtomicInteger(NOT_RUN_YET);

        private final AtomicInteger churned = new AtomicInteger(NOT_RUN_YET);

        private final AtomicInteger captchaShown = new AtomicInteger(NOT_RUN_YET);

        private final AtomicInteger captchaSolved = new AtomicInteger(NOT_RUN_YET);

        private final AtomicInteger answered = new AtomicInteger(NOT_RUN_YET);

        private final AtomicInteger datesFound = new AtomicInteger(NOT_RUN_YET);

        private final AtomicInteger errors = new AtomicInteger(NOT_RUN_YET);

        private final AtomicLong runSeconds = new AtomicLong(NOT_RUN_YET);

        private final AtomicLong lastFinishEpoch = new AtomicLong(NOT_RUN_YET);

        private volatile long startedAtMillis = System.currentTimeMillis();

        /** The day this funnel is currently accumulating; null until the first tick ever. */

        private volatile LocalDate day;

        /** Whether that day's pool has already come back empty. Keeps endDay silent after the first. */

        private volatile boolean finished;

        BrandFunnel(MeterRegistry registry, String brand) {

            Tags tags = Tags.of("brand", brand.toLowerCase());

            gauge(registry, "imei_activation_due", tags, due,

                    "Imeis outstanding at the first tick after midnight; -1 = no tick yet");

            gauge(registry, "imei_activation_churned", tags, churned,

                    "Imeis today's ticks have reached so far; catches up to due when the day clears");

            gauge(registry, "imei_activation_captcha_shown", tags, captchaShown,

                    "Captchas that rendered; collapses when the far end stops serving the widget");

            gauge(registry, "imei_activation_captcha_solved", tags, captchaSolved,

                    "Captchas broken; divide by captcha_shown for the solver's accuracy");

            gauge(registry, "imei_activation_answered", tags, answered,

                    "Imeis the portal gave a verdict for; this is what lets a row leave the queue");

            gauge(registry, "imei_activation_dates_found", tags, datesFound,

                    "Activation dates obtained; a fraction of answered by nature, not a health signal");

            gauge(registry, "imei_activation_errors", tags, errors,

                    "Exceptions during the last pass for this brand");

            Gauge.builder("imei_activation_run_seconds", runSeconds, AtomicLong::get)

                    .tags(tags)

                    .description("Wall-clock seconds of the last pass; -1 = no pass yet")

                    .register(registry);

            Gauge.builder("imei_activation_last_finish_epoch_seconds", lastFinishEpoch, AtomicLong::get)

                    .tags(tags)

                    .description("Unix time the last pass finished; staleness alerts key off this")

                    .register(registry);

        }

        private static void gauge(MeterRegistry registry, String name, Tags tags,

                                  AtomicInteger value, String description) {

            Gauge.builder(name, value, AtomicInteger::get)

                    .tags(tags)

                    .description(description)

                    .register(registry);

        }

        void reset(int dueCount) {

            startedAtMillis = System.currentTimeMillis();

            due.set(dueCount);

            churned.set(0);

            captchaShown.set(0);

            captchaSolved.set(0);

            answered.set(0);

            datesFound.set(0);

            errors.set(0);

        }

    }

}