"""Load the published historical Cypher snapshots into a scoped Neo4j graph."""
import argparse
import hashlib
import json
import os
import re
from collections import defaultdict
from pathlib import Path
from urllib.parse import quote
from urllib.request import urlopen

LIBRARY = "https://experimentsin.ai/library/sources/"
WASTE_DIR = "project-graphs/files/cypher/"
FEDERAL_DIR = "us-portfolio/files/Example_0_a_thousand_US_projects/cypher_code/"
# case id, original file, SHA-256, expected nodes and directed edges
WASTE = [
    ("tasks", "2020 02 tasks w tasks as nodes.cypher.txt", "67dd4752a5b9795d31979e012041635a6c4cd993c55df1acac5bb24a6c2bd36f", 4, 3),
    ("states", "2020 02 states as nodes only LR.cypher", "f0dff3e2a1e4fc34ec59ae27488f14f4561e7415dbc4c7108773f86e50c90de5", 4, 3),
    ("tasks-states", "2020 02 tasks and states as nodes LR.cypher.txt", "2f230b2fb3868251a36ee4fc76556847c94ac888060225f8e6b5061a48893325", 8, 7),
    ("resources", "2020 02 Tasks and states and resources LR.cypher.txt", "90730f9d5dc4bc1c61463c1f822e87f78c78d9810c407b9d637db2763ef8b8a8", 12, 12),
    ("extra-links", "2020 02 task state and resource w extra links.cypher", "e47ecc94eeb71d62dc08d748eb7e0dc429a5c3fc37aa16f72784de30742537df", 12, 15),
    ("petri", "2020 02 petri net tasks LR.cypher.txt", "4707a9b292eeb4ca858758a623e4e5fc7772a459658dc907f586f0fe35b75ac0", 6, 5),
    ("schedule", "2020 02 Project task scheduling in Neo4j from Nicole White reviewed.cypher.txt", "fd764f16e5d77377557c7dfff7145027ba100d858d7e86716671fc2f0e72729f", 14, 17),
]
FEDERAL = [
    ("health", "Health_sub_graph.cypher", "98d042889dff84864eacf2b17d319db5484b33bed76a354e6eb34196da2f9afc", 2796, 3677),
    ("complete", "Complete_graph.cypher", "c67d6b0b9fa74f8c81477d0ccaf266d0e77ba12a5af9ecca2ea8896924399095", 18728, 28771),
]
KEY = re.compile(r'("(?:[^"\\]|\\.)*")|(?:`([^`]+)`|([a-zA-Z_][\w$]*))\s*:')
F_NODE = re.compile(r'CREATE \(:(\w+):`UNIQUE IMPORT LABEL` (\{.*\})\);')
F_EDGE = re.compile(r'MATCH \(n1:`UNIQUE IMPORT LABEL`\{`UNIQUE IMPORT ID`:(\d+)\}\), \(n2:`UNIQUE IMPORT LABEL`\{`UNIQUE IMPORT ID`:(\d+)\}\) CREATE \(n1\)-\[r:(\w+)\]->\(n2\);')
FDA = "009-000005317"
FDA_PROJECT_IDS = ["2150", "2151", "2152", "2154", "2155", "2156", "2157", "2160", "2167"]


def literal(value):
    # The pinned exports use JSON values with bare or backtick-quoted map keys.
    def quote_key(match):
        return match[1] or json.dumps(match[2] or match[3]) + ":"
    return json.loads(KEY.sub(quote_key, value))


def read_source(folder, spec, directory):
    _, filename, digest, _, _ = spec
    if directory:
        raw = (directory / filename).read_bytes()
    else:
        with urlopen(LIBRARY + folder + quote(filename), timeout=60) as response:
            raw = response.read()
    if hashlib.sha256(raw).hexdigest() != digest:
        raise ValueError("Source has changed: " + filename)
    return raw.decode("utf-8")


def parse_waste(source):
    nodes, edges = [], []
    for block in source.split(";"):
        block = block.strip()
        if not block.startswith("UNWIND "):
            continue
        first, body = block.split(" AS row\n", 1)
        items = literal(first[len("UNWIND "):])
        node = re.search(r'SET n:(\w+)$', body)
        if node:
            nodes.extend((row["_id"], node[1], row["properties"]) for row in items)
        else:
            kind = re.search(r'CREATE \(start\)-\[r:(\w+)\]->\(end\)', body)
            if not kind:
                raise ValueError("Unrecognised relationship block")
            edges.extend((row["start"]["_id"], row["end"]["_id"], kind[1], row["properties"]) for row in items)
    return nodes, edges


def parse_federal(source):
    nodes, edges = [], []
    for line in source.splitlines():
        if line.startswith("CREATE (:"):
            node = F_NODE.fullmatch(line)
            if not node:
                raise ValueError("Unrecognised node record")
            properties = literal(node[2])
            nodes.append((properties.pop("UNIQUE IMPORT ID"), node[1], properties))
        elif line.startswith("MATCH (n1:"):
            edge = F_EDGE.fullmatch(line)
            if not edge:
                raise ValueError("Unrecognised relationship record")
            edges.append((int(edge[1]), int(edge[2]), edge[3], {}))
    return nodes, edges


def batches(session, query, rows, case):
    for i in range(0, len(rows), 500):
        batch = rows[i:i + 500]
        def execute(tx):
            result = tx.run(query, rows=batch, case=case).single(strict=True)
            if result["written"] != len(batch):
                raise ValueError("Write count mismatch in " + case)
        session.execute_write(execute)


def install(session, case, nodes, edges):
    by_label, by_type = defaultdict(list), defaultdict(list)
    prefix = "f382/" + case + "/"
    for key, label, props in nodes:
        by_label[label].append({"key": prefix + str(key), "props": props})
    for ordinal, (left, right, kind, props) in enumerate(edges):
        by_type[kind].append({"source": prefix + str(left), "target": prefix + str(right),
                              "ordinal": ordinal, "props": props})
    for label, rows in by_label.items():
        batches(session, f"""UNWIND $rows AS x
                MERGE (n:F382Historical {{replayKey:x.key}})
                SET n:`{label}`, n += x.props, n.replayCase=$case
                RETURN count(n) AS written""", rows, case)
    for kind, rows in by_type.items():
        # Ordinals preserve the parallel edges present in the source.
        batches(session, f"""UNWIND $rows AS x
                MATCH (a:F382Historical {{replayKey:x.source}})
                MATCH (b:F382Historical {{replayKey:x.target}})
                MERGE (a)-[r:`{kind}` {{replayOrdinal:x.ordinal}}]->(b)
                SET r += x.props, r.replayCase=$case
                RETURN count(r) AS written""", rows, case)


def check_waste(session, case):
    rows = session.run("""MATCH (t:F382Historical:Task {replayCase:$case, name:'Remove waste'})
        OPTIONAL MATCH (p:F382Historical {replayCase:$case})-[r]->(t)
        RETURN t.earliest_start AS start, t.earliest_finish AS finish,
               collect(DISTINCT [p.name, p.role]) AS inputs""", case=case).data()
    expected = {
        "tasks": (["Commission suppliers"], []),
        "states": None,
        "tasks-states": (["Suppliers ready"], []),
        "resources": (["Suppliers ready"], ["Supplier"]),
        "extra-links": (["Commission suppliers", "Suppliers ready"], ["Supplier"]),
        "petri": None,
        "schedule": (["Commission suppliers", "Suppliers ready"], ["Supplier"]),
    }[case]
    if expected is None:
        if rows:
            raise ValueError("Unexpected removal task in " + case)
        if case == "states":
            pair = session.run("""MATCH (a:F382Historical:State {replayCase:$case})-[r:transitions_with_task]->(b:F382Historical:State {replayCase:$case, name:'No waste left on site'})
                RETURN a.name AS predecessor, keys(r) AS fields""", case=case).single(strict=True)
            if pair["predecessor"] != "Suppliers ready" or set(pair["fields"]) != {"replayOrdinal", "replayCase"}:
                raise ValueError("Final state edge or its source properties have changed")
            return {"removal_task": False, "predecessor_of_final_state": pair["predecessor"]}
        transitions = session.run("""MATCH (t:F382Historical:Transition {replayCase:$case})
            RETURN t.identity AS name ORDER BY name""", case=case).value("name")
        if transitions != ["Identify waste", "Prepare Waste extraction plan"]:
            raise ValueError("The saved Petri fragment differs")
        return {"removal_task": False, "transitions": transitions}
    if len(rows) != 1:
        raise ValueError("Removal task is missing or duplicated")
    row = rows[0]
    inputs = row.pop("inputs")
    row["names"] = sorted({name for name, _ in inputs if name is not None})
    row["roles"] = sorted({role for _, role in inputs if role is not None})
    if (row["names"], row["roles"]) != expected:
        raise ValueError("Removal inputs differ in " + case)
    if (row["start"], row["finish"]) != ((40, 50) if case == "schedule" else (None, None)):
        raise ValueError("Stored time mismatch in " + case)
    return row


def check_federal(session, case):
    record = session.run("""MATCH (i:F382Historical:Investment {replayCase:$case, Identifier:$id})
        MATCH (i)-[:pays_for]->(p:F382Historical:Project {replayCase:$case})
        MATCH (i)-[:judged_by]->(m:F382Historical:Metric {replayCase:$case})
        RETURN collect(DISTINCT p.Id) AS projects, collect(DISTINCT [m.ID, m.Metric_results]) AS metrics,
               count(DISTINCT [p.Id, m.ID]) AS pairs""", case=case, id=FDA).single(strict=True)
    metrics = dict(record["metrics"])
    wanted = {"3481": "Not Met"}
    if case == "health":
        wanted.update(dict.fromkeys(["3479", "3480", "3482", "3483"]))
    if sorted(record["projects"]) != FDA_PROJECT_IDS or metrics != wanted or record["pairs"] != 9 * len(wanted):
        raise ValueError("FDA checkpoint failed in " + case)
    return {"projects": sorted(record["projects"]), "measures": metrics, "pairs": record["pairs"]}


def check(session, spec, family):
    case, _, _, n, e = spec
    qn = "MATCH (n:F382Historical {replayCase:$case}) RETURN count(n) AS n"
    qe = "MATCH (:F382Historical {replayCase:$case})-[r {replayCase:$case}]->(:F382Historical {replayCase:$case}) RETURN count(r) AS n"
    if (session.run(qn, case=case).single()["n"], session.run(qe, case=case).single()["n"]) != (n, e):
        raise ValueError("Database counts differ in " + case)
    evidence = check_waste(session, case) if family == "waste" else check_federal(session, case)
    print(f"{case}: {n} nodes; {e} directed relationships; {json.dumps(evidence, sort_keys=True)}")
    return evidence


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("family", choices=("waste", "federal"))
    parser.add_argument("--source-dir", type=Path, help="Directory containing the selected family’s original source files.")
    parser.add_argument("--check-only", action="store_true", help="Read existing graph results without importing.")
    args = parser.parse_args()
    from neo4j import GraphDatabase
    specs, folder = (WASTE, WASTE_DIR) if args.family == "waste" else (FEDERAL, FEDERAL_DIR)
    user = os.environ.get("NEO4J_USERNAME") or os.environ.get("NEO4J_USER") or "neo4j"
    settings = (os.environ["NEO4J_URI"], (user, os.environ["NEO4J_PASSWORD"]))
    with GraphDatabase.driver(settings[0], auth=settings[1]) as driver:
        driver.verify_connectivity()
        with driver.session(database=os.environ.get("NEO4J_DATABASE", "neo4j")) as session:
            if not args.check_only:
                session.run("CREATE CONSTRAINT f382_historical_replay_key IF NOT EXISTS FOR (n:F382Historical) REQUIRE n.replayKey IS UNIQUE").consume()
            results = {}
            for spec in specs:
                case, _, _, n, e = spec
                if not args.check_only:
                    content = read_source(folder, spec, args.source_dir)
                    nodes, edges = parse_waste(content) if args.family == "waste" else parse_federal(content)
                    ids = {row[0] for row in nodes}
                    if len(ids) != n or len(nodes) != n or len(edges) != e or any(a not in ids or b not in ids for a,b,_,_ in edges):
                        raise ValueError("Source identities or counts differ in " + case)
                    install(session, case, nodes, edges)
                results[case] = check(session, spec, args.family)
            if args.family == "federal" and results["health"]["projects"] != results["complete"]["projects"]:
                raise ValueError("FDA project identities differ between exports")


if __name__ == "__main__":
    main()
