#!/usr/bin/env node
/*
 * Peeking at a test: how often two identical versions produce a winner.
 *
 * The simulation behind "Called once, at the planned end" in /writing/calling-a-test. It runs
 * 40,000 fourteen-day tests in which both versions convert at exactly 3%, on 500 clicks per
 * version per day, and judges each with the product's rule: a one-sided test of two proportions
 * (the same formula as `confidence` in packages/core/src/experiments.ts), at least 20 conversions
 * per version, and a winner at 95% confidence or more. It calls each test two ways:
 *
 *   - checking every day and stopping the first day the new version reaches 95%;
 *   - looking once, when the planned 14 days end, as the product does.
 *
 * The versions are identical, so every winner is a false one. The random numbers are seeded, so a
 * run prints the same results every time. It needs only Node:
 *
 *   node peeking.mjs              40,000 tests, seed 1
 *   node peeking.mjs 100000 7     100,000 tests, seed 7
 */

const [trials = 40000, seed = 1] = process.argv.slice(2).map(Number);
const DAYS = 14;
const CLICKS_PER_DAY = 500;
const RATE = 0.03;
const CALL_AT = 0.95;
const MIN_CONVERSIONS = 20;

/** A small seeded generator (mulberry32): uniform numbers in [0, 1). */
function random(s) {
  let a = s >>> 0;
  return () => {
    a = (a + 0x6d2b79f5) | 0;
    let t = Math.imul(a ^ (a >>> 15), 1 | a);
    t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
  };
}

/** One day's conversions for one version: a draw from Binomial(500, 3%), by inverting its distribution. */
const cdf = [];
for (let k = 0, p = (1 - RATE) ** CLICKS_PER_DAY, c = 0; k <= CLICKS_PER_DAY; k++) {
  c += p;
  cdf.push(c);
  p *= ((CLICKS_PER_DAY - k) / (k + 1)) * (RATE / (1 - RATE));
}
function conversions(u) {
  let lo = 0;
  let hi = CLICKS_PER_DAY;
  while (lo < hi) {
    const mid = (lo + hi) >> 1;
    if (cdf[mid] < u) lo = mid + 1;
    else hi = mid;
  }
  return lo;
}

/** Standard normal CDF (Abramowitz and Stegun 7.1.26), as in packages/core/src/experiments.ts. */
function phi(z) {
  const t = 1 / (1 + 0.3275911 * (Math.abs(z) / Math.SQRT2));
  const y = 1 - ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t * Math.exp(-(z * z) / 2);
  return z >= 0 ? (1 + y) / 2 : (1 - y) / 2;
}

/** One minus the one-sided p-value that the new version converts better than the current one. */
function confidence(newConv, curConv, clicks) {
  const pooled = (newConv + curConv) / (2 * clicks);
  const se = Math.sqrt(pooled * (1 - pooled) * (2 / clicks));
  return se ? phi((newConv - curConv) / clicks / se) : 0.5;
}

const wins = (cur, next, clicks) => cur >= MIN_CONVERSIONS && next >= MIN_CONVERSIONS && confidence(next, cur, clicks) >= CALL_AT;

const rand = random(seed);
let stoppedEarly = 0;
let calledAtEnd = 0;
for (let t = 0; t < trials; t++) {
  let cur = 0;
  let next = 0;
  let early = false;
  for (let day = 1; day <= DAYS; day++) {
    cur += conversions(rand());
    next += conversions(rand());
    if (!early && wins(cur, next, day * CLICKS_PER_DAY)) early = true;
  }
  if (early) stoppedEarly++;
  if (wins(cur, next, DAYS * CLICKS_PER_DAY)) calledAtEnd++;
}

const share = (n) => `${n.toLocaleString("en-US")} of ${trials.toLocaleString("en-US")}, ${((n / trials) * 100).toFixed(1)}%, about one time in ${Math.round(trials / n)}`;
console.log(`${trials.toLocaleString("en-US")} fourteen-day tests of two identical versions, ${CLICKS_PER_DAY} clicks per version per day, ${RATE * 100}% conversion (seed ${seed}).`);
console.log(`Checking daily, stopping at the first day over 95%: a false winner in ${share(stoppedEarly)}.`);
console.log(`Looking once, at the planned end:                    a false winner in ${share(calledAtEnd)}.`);