#!/usr/bin/env python3
"""Compare native and Graal profiles using SQLite's upstream speedtest1."""

import argparse
import hashlib
import json
import math
import os
from pathlib import Path
import platform
import re
import signal
from statistics import median
import subprocess
import sys
import time


ROOT = Path(__file__).resolve().parents[1]
GRAPH_CAP = 150000
LIFTED_CAP = 2147483647
VERIFY_HASH = re.compile(r"(?m)^Verification Hash: ([0-9]+) ([0-9a-f]+)$")
TOTAL = re.compile(r"(?m)^ *TOTAL\.+ +(\d+\.\d{3})s$")
TEST_TIME = re.compile(r"(?m)^ *(\d+) - ([^\n]{60}) +(\d+\.\d{3})s$")
PROFILE_NAME = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9_-]*")
VDBE_DONE = re.compile(
    r"(?m)^\[engine\] opt done[^\n]*\bsqlite3VdbeExec\s+\|Tier\s+(\d+)"
    r"[^\n]*\bCodeSize\s+([1-9]\d*)"
)
VDBE_SIZE_BAILOUT = re.compile(
    r"(?m)^\[engine\] opt failed[^\n]*\bsqlite3VdbeExec\s+\|Tier[^\n]*"
    r"\bGraphTooBigBailoutException\b"
)


def kill_group(pid):
    try:
        os.killpg(pid, signal.SIGKILL)
    except ProcessLookupError:
        pass


def resident_high_water(pid):
    try:
        with Path(f"/proc/{pid}/status").open("rb") as status:
            for line in status:
                if line.startswith(b"VmHWM:"):
                    return int(line.split()[1])
    except (FileNotFoundError, ProcessLookupError):
        pass
    return 0


def measure(command, directory, environment, timeout):
    with (directory / "stdout.log").open("wb") as out, (directory / "stderr.log").open("wb") as err:
        started = time.monotonic()
        with subprocess.Popen(command, cwd=directory, env=environment, stdout=out, stderr=err,
                              start_new_session=True) as child:
            timed_out = False
            post_exec_peak = 0
            try:
                while True:
                    pid, status, usage = os.wait4(child.pid, os.WNOHANG)
                    if pid:
                        break
                    post_exec_peak = max(post_exec_peak, resident_high_water(child.pid))
                    if time.monotonic() - started >= timeout:
                        timed_out = True
                        kill_group(child.pid)
                        _, status, usage = os.wait4(child.pid, 0)
                        break
                    time.sleep(0.001)
                wall_seconds = time.monotonic() - started
                child.returncode = os.waitstatus_to_exitcode(status)
            finally:
                if child.returncode is None:
                    kill_group(child.pid)
                    child.wait()

    output = (directory / "stdout.log").read_text(errors="replace")
    trace = (directory / "stderr.log").read_text(errors="replace")
    digest = VERIFY_HASH.search(output)
    total = TOTAL.search(output)
    return {
        "exit_code": child.returncode,
        "timed_out": timed_out,
        "wall_seconds": wall_seconds,
        "peak_rss_kib": usage.ru_maxrss,
        "post_exec_peak_rss_sample_kib": post_exec_peak or None,
        "user_cpu_seconds": usage.ru_utime,
        "system_cpu_seconds": usage.ru_stime,
        "speedtest_seconds": float(total[1]) if total else None,
        "speedtest_cases": [
            {"case": int(case), "name": name.rstrip("."), "seconds": float(seconds)}
            for case, name, seconds in TEST_TIME.findall(output)
        ],
        "verification_count": int(digest[1]) if digest else None,
        "verification_hash": digest[2] if digest else None,
        "vdbe_entry_compilations": [
            {"tier": int(tier), "code_bytes": int(size)} for tier, size in VDBE_DONE.findall(trace)
        ],
        "vdbe_graph_size_bailouts": len(VDBE_SIZE_BAILOUT.findall(trace)),
    }


def save_report(output, report):
    temporary = output / "report.json.tmp"
    temporary.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
    temporary.replace(output / "report.json")


def file_record(path):
    digest = hashlib.sha256()
    with path.open("rb") as stream:
        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
            digest.update(chunk)
    return {"path": str(path), "sha256": digest.hexdigest()}


def resolve_artifact(path):
    result = path.resolve(strict=True)
    if not result.is_file():
        raise ValueError(f"artifact is not a file: {result}")
    return result


def profile_spec(spec, option):
    name, separator, value = spec.partition("=")
    if not separator or not value or not PROFILE_NAME.fullmatch(name):
        raise ValueError(f"{option} expects PROFILE=VALUE, got {spec!r}")
    return name, value


def profile_paths(specs, option):
    paths = {}
    for spec in specs:
        name, path = profile_spec(spec, option)
        if name in paths:
            raise ValueError(f"{option} repeated for profile {name}")
        paths[name] = resolve_artifact(Path(path))
    return paths


def command_profiles(args):
    inputs = {
        "native_o0": resolve_artifact(args.native_o0),
        "native_o2": resolve_artifact(args.native_o2),
        "runner": resolve_artifact(args.runner),
    }
    commands = {name: [str(inputs[name.replace("-", "_")])] for name in ("native-o0", "native-o2")}
    if args.bitcode:
        inputs["bitcode"] = resolve_artifact(args.bitcode)
        modules = {name: inputs["bitcode"] for name in ("graal-capped", "graal-lifted")}
    else:
        modules = profile_paths(args.graal, "--graal")
        inputs.update({f"{name}:bitcode": path for name, path in modules.items()})
    if modules.keys() & commands.keys():
        raise ValueError("Graal profiles must have distinct names from the native profiles")
    runners = profile_paths(args.graal_runner, "--graal-runner")
    if runners.keys() - modules.keys():
        raise ValueError("--graal-runner specified for an unknown Graal profile")
    profile_flags = {name: [] for name in modules}
    for spec in args.profile_runtime_arg:
        name, flag = profile_spec(spec, "--profile-runtime-arg")
        if name not in profile_flags or not flag.startswith("-"):
            raise ValueError("--profile-runtime-arg expects a Graal profile and a flag starting with '-'")
        profile_flags[name].append(flag)
    for name, module in modules.items():
        runner = runners.get(name, inputs["runner"])
        if name in runners:
            inputs[f"{name}:runner"] = runner
        image = runner.parent / "libexec" / "sqlite-api-shell-image"
        if image.is_file():
            key = f"{name}:runner-image" if name in runners else "runner_image"
            inputs[key] = image.resolve()
        cap = GRAPH_CAP if name == "graal-capped" else LIFTED_CAP
        background = "false" if args.synchronous else "true"
        commands[name] = [
            str(runner), f"-Dpolyglot.compiler.MaximumGraalGraphSize={cap}",
            f"-Dpolyglot.engine.BackgroundCompilation={background}", *args.runtime_arg,
            *profile_flags[name],
            "--run-module", "--trace-compilation", f"--libraries={args.libraries}", str(module),
        ]
    return inputs, commands


def rotate(items, offset):
    offset %= len(items)
    return items[offset:] + items[:offset]


def valid_trial(trial):
    total = trial["speedtest_seconds"]
    cases = trial["speedtest_cases"]
    return (
        trial["exit_code"] == 0 and not trial["timed_out"]
        and trial["verification_count"] and trial["verification_hash"]
        and total is not None and cases
        and math.isclose(total, sum(case["seconds"] for case in cases), abs_tol=1e-6)
    )


def summarise_testset(trials, profiles):
    native = next(trial for trial in trials if trial["profile"] == "native-o2")
    expected = native["verification_count"], native["verification_hash"]
    case_names = [(case["case"], case["name"]) for case in native["speedtest_cases"]]
    matches = all(
        valid_trial(trial)
        and (trial["verification_count"], trial["verification_hash"]) == expected
        and [(case["case"], case["name"]) for case in trial["speedtest_cases"]] == case_names
        for trial in trials
    )
    summary = {
        "verification_matches": matches,
        "native_o2_verification_count": expected[0],
        "native_o2_verification_hash": expected[1],
    }
    if matches:
        modes = {}
        for profile in profiles:
            runs = [trial for trial in trials if trial["profile"] == profile]
            wall_times = [trial["wall_seconds"] for trial in runs]
            test_times = [trial["speedtest_seconds"] for trial in runs]
            modes[profile] = {
                "wall_median_s": median(wall_times),
                "wall_min_s": min(wall_times),
                "wall_max_s": max(wall_times),
                "speedtest_median_s": median(test_times),
                "speedtest_min_s": min(test_times),
                "speedtest_max_s": max(test_times),
            }
        summary["profiles"] = modes
    return summary


def benchmark(args):
    inputs, profiles = command_profiles(args)
    testsets = args.testset or ["main"]
    commands = {
        testset: {
            profile: [*prefix, "--size", str(args.size), "--testset", testset, "--verify", "--memdb"]
            for profile, prefix in profiles.items()
        } for testset in testsets
    }
    environment = dict(os.environ, CGRAAL_INTERPRET="0", CGRAAL_TRACE="0")
    report = {
        "platform": platform.platform(),
        "cpu_affinity": sorted(os.sched_getaffinity(0)),
        "measurement": {
            "wall_seconds": "monotonic process elapsed time including startup, with 1 ms exit polling",
            "peak_rss_kib": "Linux wait4 high water mark, including possible pre-exec launcher RSS",
            "post_exec_peak_rss_sample_kib": "largest observed VmHWM sample after exec, a lower bound",
        },
        "inputs": {name: file_record(path) for name, path in inputs.items()},
        "testsets": testsets,
        "commands_by_testset": commands,
        "timeout_seconds": args.timeout,
        "planned_trials_per_profile": args.trials,
        "trials": [],
    }
    if len(testsets) == 1:
        report["commands"] = commands[testsets[0]]
    output = args.output.resolve()
    output.mkdir(parents=True)
    save_report(output, report)

    for index in range(args.trials):
        for testset in rotate(testsets, index):
            for profile in rotate(list(profiles), index + testsets.index(testset)):
                relative = Path(profile) / str(index + 1)
                if len(testsets) > 1:
                    relative = Path(testset) / relative
                directory = output / relative
                directory.mkdir(parents=True)
                result = measure(commands[testset][profile], directory, environment, args.timeout)
                report["trials"].append({"testset": testset, "profile": profile, "trial": index + 1,
                                         "logs": str(relative), **result})
                save_report(output, report)
                print(f"{testset}/{profile} #{index + 1}  {result['wall_seconds']:.4f} s  "
                      f"{result['peak_rss_kib']} KiB RSS  exit {result['exit_code']}", flush=True)

    report["summary"] = {
        testset: summarise_testset([t for t in report["trials"] if t["testset"] == testset], profiles)
        for testset in testsets
    }
    valid = all(summary["verification_matches"] for summary in report["summary"].values())
    report["verification_matches"] = valid
    save_report(output, report)
    print(output / "report.json")
    if not valid:
        raise ValueError("a run failed, timings were incomplete or verification outputs differ")


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--native-o0", type=Path, required=True)
    parser.add_argument("--native-o2", type=Path, required=True)
    modes = parser.add_mutually_exclusive_group(required=True)
    modes.add_argument("--bitcode", type=Path, help="matched -O0 bitcode for the two original Graal profiles")
    modes.add_argument("--graal", action="append", metavar="PROFILE=BITCODE",
                       help="run one named Graal profile, repeat for additional profiles")
    parser.add_argument("--runner", type=Path, default=ROOT / "dist" / "sqlite-api-shell")
    parser.add_argument("--graal-runner", action="append", default=[], metavar="PROFILE=PATH",
                        help="use a distinct runtime launcher for a named Graal profile")
    parser.add_argument("--runtime-arg", action="append", default=[], metavar="FLAG",
                        help="forward one runtime flag to all Graal profiles")
    parser.add_argument("--profile-runtime-arg", action="append", default=[], metavar="PROFILE=FLAG",
                        help="forward one runtime flag after the common flags for the selected Graal profile")
    parser.add_argument("--synchronous", action="store_true", help="compile Graal guest code synchronously")
    parser.add_argument("--testset", action="append", choices=("main", "cte", "orm"),
                        help="run this upstream testset in fresh processes, repeat to select more sets")
    parser.add_argument("--output", type=Path, required=True, help="new directory for raw logs and report.json")
    parser.add_argument("--trials", type=int, default=3)
    parser.add_argument("--size", type=int, default=1, help="upstream speedtest1 size parameter")
    parser.add_argument("--timeout", type=float, default=180, help="seconds per trial")
    parser.add_argument("--libraries", default="libz.so.1:libm.so.6")
    args = parser.parse_args()
    if sys.platform != "linux":
        parser.error("peak RSS measurement requires Linux")
    if args.trials < 1 or args.size < 1 or not math.isfinite(args.timeout) or args.timeout <= 0:
        parser.error("trials and size must be positive and timeout must be finite and positive")
    if args.testset and len(set(args.testset)) != len(args.testset):
        parser.error("each testset can be selected once")
    try:
        benchmark(args)
    except (OSError, ValueError) as failure:
        print(f"benchmark-speedtest: {failure}", file=sys.stderr)
        return 1
    return 0


if __name__ == "__main__":
    sys.exit(main())
