"""
Replay real Binance USD-M futures trades through the SAME bot code, and report
what the strategy actually did — including how many bricks the 1hr chart runs
before returning to the line (your "it printed 50 or more" claim).

  python backtest.py --download 2026-11-06 2026-11-08   # fetch daily aggTrades
  python backtest.py --data data --both                 # fees off, then on
  python backtest.py --data data --both --fill market   # realistic fills

'line'  = the ideal case, the flip fills exactly on the line.
'market'= the flip fills at the trade price + 1 tick.
Funding, latency and order-book depth are not modelled.
"""
import argparse
import asyncio
import contextlib
import csv
import glob
import io
import os
import statistics
import sys
import urllib.error
import urllib.request
import zipfile
from collections import Counter
from datetime import datetime, timedelta, timezone
from decimal import Decimal as D

import zec_renko_bot as z

URL = ("https://data.binance.vision/data/futures/um/daily/aggTrades/"
       "{s}/{s}-aggTrades-{d}.zip")


def download(start, end, folder, symbol):
    os.makedirs(folder, exist_ok=True)
    day = datetime.strptime(start, "%Y-%m-%d").date()
    last = datetime.strptime(end, "%Y-%m-%d").date()
    while day <= last:
        name = f"{symbol}-aggTrades-{day}.zip"
        path = os.path.join(folder, name)
        if os.path.exists(path) and zipfile.is_zipfile(path):
            print("have", name)
        else:
            tmp = path + ".part"
            try:
                urllib.request.urlretrieve(URL.format(s=symbol, d=day), tmp)
                if not zipfile.is_zipfile(tmp):
                    raise ValueError("download is not a valid zip (truncated?)")
                os.replace(tmp, path)
                print("downloaded", name, os.path.getsize(path) // 1024 // 1024, "MB")
            except urllib.error.HTTPError as e:
                print("missing", name, e.code, "(not published or wrong date)")
            except Exception as e:
                print("failed", name, e)
        day += timedelta(days=1)


def iter_ticks(path):
    """Yield (price Decimal, timestamp ms) from a zipped or plain aggTrades file."""
    if path.endswith(".zip"):
        zf = zipfile.ZipFile(path)
        f = io.TextIOWrapper(zf.open(zf.namelist()[0]), encoding="utf-8")
    else:
        f = open(path, encoding="utf-8")
    with f:
        for row in csv.reader(f):
            try:
                ts = int(row[5])
            except (ValueError, IndexError):
                continue                      # header or bad line
            if ts > 10 ** 14:
                ts //= 1000
            yield D(row[1]), ts


class Replay(z.Bot):
    """The live bot fed from files instead of a websocket."""

    def __init__(self, fee, fill, capital):
        z.TAKER_FEE = D(str(fee))
        z.FILL = fill
        z.START_CASH = D(str(capital))
        super().__init__(None)
        self.rows = []
        self.cur_ts = 0
        self.prev_price = None
        self.st = dict(trades=0, win=0, loss=0, neutral=0, cycles=0, profit=0,
                       lines=0, fees=D(0), funding=D(0), spread=D(0), net=D(0),
                       liquidated=False)
        self.peak = self.capital
        self.max_dd = D(0)
        self.runs = []            # length of every same-direction run of 1hr bricks
        self.run = 0
        self.last_dir = 0
        self.held = 0             # 1hr bricks printed while a trade was held = the profit
        self.profit_bricks = []   # that count, per booked cycle
        self.cycle_bricks = []    # each booked cycle, measured in 0.1% bricks
        self.flips = []           # trades per cycle
        self._spread_seen = D(0)

    def load(self):
        pass

    def save(self):
        pass

    def log(self, event, detail="", pnl="", fees=""):
        t = datetime.fromtimestamp(self.cur_ts / 1000, timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
        self.rows.append([t, event, detail, pnl, fees,
                          f"{self.capital:.6f}", f"{self.savings:.6f}"])
        s = self.st
        if event.startswith("3rd chart"):
            s["trades"] += 1
            s["win" if "win" in event else "loss" if "loss" in event else "neutral"] += 1
        elif event == "NEW LINE":
            s["lines"] += 1

    def set_line(self, line, side):
        super().set_line(line, side)
        if self.pos and self.pos["carried"]:
            self.held += 1        # every 1hr brick the trend prints is ours
        if side == self.last_dir:
            self.run += 1
        else:
            if self.run:
                self.runs.append(self.run)
            self.run, self.last_dir = 1, side

    def on_close(self, net, base, move):
        self.st["net"] += net
        equity = self.capital + self.savings
        self.peak = max(self.peak, equity)
        if self.peak > 0:
            self.max_dd = max(self.max_dd, (self.peak - equity) / self.peak)

    def on_book(self, total, fees, n, base):
        s = self.st
        s["cycles"] += 1
        s["fees"] += fees
        if total > 0:
            s["profit"] += 1
        brick = base * z.MARGIN_USE * D(str(z.LEVERAGE)) * z.BOX_HOURLY
        if brick > 0:
            self.cycle_bricks.append(float(total / brick))
        self.profit_bricks.append(self.held)
        self.held = 0
        self.flips.append(n)

    def liquidated(self, price):
        if super().liquidated(price):
            self.st["liquidated"] = True
            return True
        return False

    async def on_trade(self, price, ts):
        self.cur_ts = ts
        self.prev_price = self.last_price
        await super().on_trade(price, ts)

    def close(self, raw, slip, reason):
        p = self.pos
        if p and not z.MAKER:                  # report the true crossing cost of the round trip
            self.st["spread"] += (p["entry"] * p["qty"] + raw * p["qty"]) * z.SPREAD / 2
        super().close(raw, slip, reason)

    def accrue_funding(self, price):
        before = self.cycle.get("funding", D(0))
        super().accrue_funding(price)
        self.st["funding"] += self.cycle.get("funding", D(0)) - before

    async def new_hour(self):
        # the hour that just closed ended at the previous trade's price
        if self.prev_price is None:
            return
        bricks = self.r1h.feed(self.prev_price)
        for b in bricks:                     # every brick advances the line, not just the last
            self.set_line(b[0], b[2])


def bucket(values, edges=(1, 2, 3, 5, 10, 20, 50)):
    c = Counter()
    for v in values:
        for e in reversed(edges):
            if v >= e:
                c[f"{e}+"] += 1
                break
        else:
            c["<1"] += 1
    return " ".join(f"{k}:{c[k]}" for k in ["<1"] + [f"{e}+" for e in reversed(edges)] if c[k])


def run(files, fee, fill, capital):
    bot = Replay(fee, fill, capital)
    ticks = 0
    first = last = None

    async def go():
        nonlocal ticks, first, last
        for path in files:
            for price, ts in iter_ticks(path):
                if first is None:
                    first = ts
                last = ts
                ticks += 1
                await bot.on_trade(price, ts)
                if bot.dead:
                    return
            sys.__stdout__.write(f"  {os.path.basename(path)} done | equity "
                                 f"{bot.capital + bot.savings:.4f}\n")

    with open(os.devnull, "w") as dn, contextlib.redirect_stdout(dn):
        asyncio.run(go())

    tag = f"fee{fee}_{fill}_{z.GATE_MODE}_t{z.TRAIL_BRICKS}_fu{z.FUNDING_PER_HOUR}"
    out = f"bt_events_{tag}.csv"
    with open(out, "w", newline="") as f:
        w = csv.writer(f)
        w.writerow(["time", "event", "detail", "pnl", "fees", "capital", "savings"])
        w.writerows(bot.rows)

    s = bot.st
    fmt = lambda ms: datetime.fromtimestamp(ms / 1000, timezone.utc).strftime("%Y-%m-%d %H:%M")
    equity = bot.capital + bot.savings
    print(f"\n=== RESULT | fee {float(fee) * 100:.3f}% per order | fills: {fill} ===")
    print(f"Period: {fmt(first)} -> {fmt(last)} UTC | ticks: {ticks:,}")
    print(f"1hr lines: {s['lines']} | cycles: {s['cycles']} (profitable {s['profit']})")
    if s["cycles"]:
        print(f"trades per cycle: avg {statistics.mean(bot.flips):.1f} | "
              f"median {statistics.median(bot.flips):.0f} | max {max(bot.flips)}")
    avg = s["trades"] / s["cycles"] if s["cycles"] else 0
    print(f"chart-3 trades: {s['trades']} (avg {avg:.1f}/cycle) | "
          f"win {s['win']} / loss {s['loss']} / neutral {s['neutral']}")
    print(f"fees paid: {s['fees']:.4f} | spread paid: {s['spread']:.4f} | "
          f"funding paid: {s['funding']:.4f} (all inside net) | net: {s['net']:.4f} | "
          f"gross before costs: "
          f"{s['net'] + s['fees'] + s['funding'] + s['spread']:.4f}")
    print(f"start {capital} -> cash {bot.capital:.4f} + savings {bot.savings:.4f} "
          f"= {equity:.4f} | max drawdown {bot.max_dd * 100:.1f}%")
    if s["liquidated"]:
        print("LIQUIDATED during this run.")
    runs = bot.runs + ([bot.run] if bot.run else [])
    if runs:
        print(f"1hr brick runs: n={len(runs)} | longest {max(runs)} | "
              f"median {statistics.median(runs):.0f} | mean {statistics.mean(runs):.2f}")
        print(f"  run lengths   {bucket(runs)}")
    if bot.profit_bricks:
        pb = bot.profit_bricks
        brick_value = float(D(str(capital)) * z.MARGIN_USE * z.LEVERAGE * z.BOX_HOURLY)
        print(f"1hr bricks printed while holding (the profit): total {sum(pb)} | "
              f"per cycle median {statistics.median(pb):.0f} | mean {statistics.mean(pb):.2f} | max {max(pb)}")
        print(f"  = {sum(pb) * brick_value:.2f} USD of gross if every brick is kept "
              f"(1 brick = {brick_value:.4f} USD at {z.LEVERAGE}x)")
    if bot.cycle_bricks:
        cb = bot.cycle_bricks
        good = sum(1 for x in cb if x >= 1)
        print(f"bricks per booked cycle, NET of fees: "
              f"median {statistics.median(cb):+.2f} | mean {statistics.mean(cb):+.2f} | "
              f"best {max(cb):+.2f} | worst {min(cb):+.2f} | cycles >=1 brick: {good}/{len(cb)}")
    print(f"trade log: {out}\n")


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--download", nargs=2, metavar=("START", "END"))
    ap.add_argument("--data", default="data")
    ap.add_argument("--symbol", default="ZECUSDT")
    ap.add_argument("--fee", type=float, default=0.0005)
    ap.add_argument("--fill", choices=["line", "market"], default="line")
    ap.add_argument("--capital", type=float, default=1)
    ap.add_argument("--leverage", type=float, default=75)
    ap.add_argument("--gate", choices=["touch3", "chart1", "band"], default="touch3")
    ap.add_argument("--trail", type=int, default=0, help="bricks the exit trails behind the newest line")
    ap.add_argument("--maker", action="store_true",
                    help="rest limit orders at the line: no spread cost (assumes every fill)")
    ap.add_argument("--spread", type=float, default=0.000085,
                    help="bid-ask spread as a fraction of price (0.000085 = 0.0085%%)")
    ap.add_argument("--funding", type=float, default=0.000012,
                    help="funding per hour of open notional (default 0.0012%%/h)")
    ap.add_argument("--both", action="store_true", help="run with fees off, then on")
    a = ap.parse_args()

    if a.download:
        download(a.download[0], a.download[1], a.data, a.symbol)
        return
    z.SPREAD = D(str(a.spread))
    z.MAKER = bool(a.maker)
    z.FUNDING_PER_HOUR = D(str(a.funding))
    z.LEVERAGE = int(a.leverage)
    z.GATE_MODE = a.gate
    z.TRAIL_BRICKS = int(a.trail)
    print(f"leverage {a.leverage}x | gate {a.gate} | fill {a.fill} | trail {a.trail} | capital {a.capital}")
    files = sorted(glob.glob(os.path.join(a.data, "*.zip")) +
                   glob.glob(os.path.join(a.data, "*.csv")))
    if not files:
        sys.exit(f"No .zip/.csv files in '{a.data}'. Run --download first.")
    print(f"{len(files)} files")
    for fee in ([0, a.fee] if a.both else [a.fee]):
        run(files, fee, a.fill, a.capital)


if __name__ == "__main__":
    main()