#!/usr/bin/env python3 """GSCDB holdout errors for MACE α / Δ checkpoints.""" from __future__ import annotations import argparse import hashlib import json import os import sys from pathlib import Path import pandas as pd import torch from tqdm import tqdm ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT / "common")) import datasets as utils # noqa: E402 from common.utils import CONFIGS, DATA_CSV, MODELS, default_data_root, normalize_reaction_key # noqa: E402 from mace_alpha.model import MACEAlphaHFReadout # noqa: E402 from mace_alpha.train import _parse_xyz, forward_reaction_batch_alpha_hf # noqa: E402 from mace_delta.model import MACEDeltaB3LYPSkala, forward_reaction_batch_delta_b3lyp_skala # noqa: E402 HARTREE2KCAL = 627.509474 TM_PREFIXES = ("3d4dIPSS", "CUAGAU", "DAPD", "MME", "MOBH", "ROST", "TMD", "MOR", "TMB") def parse_xyz(xyz: str): elems, coords = [], [] for line in xyz.splitlines(): p = line.split() if len(p) != 4: continue elems.append(p[0]) coords.append([float(p[1]), float(p[2]), float(p[3])]) return elems, coords def _get_first(mol: dict, *keys): for k in keys: if k in mol: return mol[k] return None def xc_b3lyp(mol: dict) -> float | None: v = _get_first(mol, "E_xc_b3lyp", "E_xc_B3LYP") if v is not None: return float(v) ex = _get_first(mol, "E_x_B3LYP", "E_x_b3lyp") ec = _get_first(mol, "E_c_B3LYP", "E_c_b3lyp") if ex is not None and ec is not None: return float(ex) + float(ec) return None def x_c_b3lyp(mol: dict) -> tuple[float, float] | None: ex = _get_first(mol, "E_x_B3LYP", "E_x_b3lyp") ec = _get_first(mol, "E_c_B3LYP", "E_c_b3lyp") if ex is None or ec is None: return None return float(ex), float(ec) def molecule_uid_delta(mol: dict, exc: float) -> str: payload = { "xyz": mol["xyz"], "E_unxc": float(mol["E_unxc"]), "hf_x": float(mol.get("hf_x", 0.0)), "d3bj_b3lyp": float(mol.get("d3bj_b3lyp", 0.0)), "E_xc_b3lyp": float(exc), } return hashlib.sha1(json.dumps(payload, sort_keys=True).encode()).hexdigest() def molecule_uid_alpha(mol: dict, ex: float, ec: float) -> str: payload = { "xyz": mol["xyz"], "E_unxc": float(mol["E_unxc"]), "hf_x": float(mol.get("hf_x", 0.0)), "d3bj_b3lyp": float(mol.get("d3bj_b3lyp", 0.0)), "E_x_B3LYP": float(ex), "E_c_B3LYP": float(ec), } return hashlib.sha1(json.dumps(payload, sort_keys=True).encode()).hexdigest() def build_cache_delta(dataset, reaction_keys: list[str]) -> tuple[dict, list[dict], int]: molecules: dict = {} reactions: list[dict] = [] skipped = 0 by_str = {str(x): x for x in dataset.keys} for key in reaction_keys: rid = by_str.get(str(key)) if rid is None: skipped += 1 continue rxn = dataset[rid] if "mols" not in rxn or not rxn["mols"] or "energy" not in rxn: skipped += 1 continue rec = {"name": str(rxn.get("name", rid)), "energy_kcal": float(rxn["energy"]), "mols": []} ok = True for mol in rxn["mols"].values(): if any(r not in mol for r in ("xyz", "E_unxc", "hf_x", "d3bj_b3lyp")): ok = False break exc = xc_b3lyp(mol) if exc is None: ok = False break elems, coords = parse_xyz(mol["xyz"]) if not elems: ok = False break uid = molecule_uid_delta(mol, exc) if uid not in molecules: molecules[uid] = { "elements": elems, "coords": coords, "E_unxc": float(mol["E_unxc"]), "hf_x": float(mol["hf_x"]), "d3bj_b3lyp": float(mol["d3bj_b3lyp"]), "E_xc_b3lyp": float(exc), } rec["mols"].append({"mol_id": uid, "coeff": float(mol.get("coeff", 1.0))}) if ok and rec["mols"]: reactions.append(rec) else: skipped += 1 return molecules, reactions, skipped def build_cache_alpha(dataset, reaction_keys: list[str]) -> tuple[dict, list[dict], int]: molecules: dict = {} reactions: list[dict] = [] skipped = 0 by_str = {str(x): x for x in dataset.keys} for key in reaction_keys: rid = by_str.get(str(key)) if rid is None: skipped += 1 continue rxn = dataset[rid] if not rxn.get("mols") or "energy" not in rxn: skipped += 1 continue rec = {"name": str(rxn.get("name", rid)), "energy_kcal": float(rxn["energy"]), "mols": []} ok = True for mol in rxn["mols"].values(): if any(r not in mol for r in ("xyz", "E_unxc", "hf_x", "d3bj_b3lyp")): ok = False break xc = x_c_b3lyp(mol) if xc is None: ok = False break ex, ec = xc elems, coords = _parse_xyz(mol["xyz"]) if not elems: ok = False break uid = molecule_uid_alpha(mol, ex, ec) if uid not in molecules: molecules[uid] = { "elements": elems, "coords": coords, "E_unxc": float(mol["E_unxc"]), "hf_x": float(mol["hf_x"]), "d3bj_b3lyp": float(mol["d3bj_b3lyp"]), "E_x_B3LYP": float(ex), "E_c_B3LYP": float(ec), } rec["mols"].append({"mol_id": uid, "coeff": float(mol.get("coeff", 1.0))}) if ok and rec["mols"]: reactions.append(rec) else: skipped += 1 return molecules, reactions, skipped def _load_ckpt(cls, path, device): model = cls(device=str(device), dtype=torch.float32) try: state = torch.load(path, map_location=device, weights_only=True) except TypeError: state = torch.load(path, map_location=device) model.load_state_dict(state, strict=False) model.eval() return model def _filter_z(rxns, molecules, allowed_z): from ase.data import atomic_numbers as Z_OF out = [] for r in rxns: if all(Z_OF[e] in allowed_z for item in r["mols"] for e in molecules[item["mol_id"]]["elements"]): out.append(r) return out @torch.no_grad() def _errors(forward, rxns, mols, model, device, batch_size=4): out = {} for start in tqdm(range(0, len(rxns), batch_size), desc="eval", leave=True): batch = rxns[start : start + batch_size] pred_h, ref_h = forward(batch, mols, model, float(model.r_max), device, torch.float32) for rxn, e in zip(batch, (pred_h - ref_h).mul(HARTREE2KCAL).cpu().numpy()): out[normalize_reaction_key(rxn["name"])] = float(e) return out def holdout_keys_from_bench(gscdb) -> list[str]: bench = DATA_CSV / "bench_holdout_melted.csv" if bench.is_file(): melt = pd.read_csv(bench) keys = ( melt.loc[melt["Method"].astype(str) == "Skala", "Reaction"] .map(normalize_reaction_key) .astype(str) .unique() .tolist() ) available = {normalize_reaction_key(k) for k in gscdb.keys} return [k for k in keys if k in available] sp = json.loads((CONFIGS / "reaction_splits_skala_finetune.json").read_text())["splits"] train_val = set(map(str, sp["gscdb_train"])) | set(map(str, sp["gscdb_val"])) errors = pd.read_csv(Path(os.environ["GSCDB_ROOT"]) / "Analysis" / "Results_errors.csv") available = set(errors["Reaction"].astype(str)) keys = [] for k in gscdb.keys: ds = str(gscdb.df.loc[k, "Dataset"]) if any(ds.startswith(p) for p in TM_PREFIXES) or ds == "OEEF": continue if str(k) in available and str(k) not in train_val: keys.append(str(k)) return keys def main(): ap = argparse.ArgumentParser() ap.add_argument("--alpha-ckpt", default=str(MODELS / "mace_alpha_hf_best.pt")) ap.add_argument("--delta-ckpt", default=str(MODELS / "mace_delta_b3lyp_best.pt")) ap.add_argument("--out-dir", default=str(ROOT / "data")) ap.add_argument("--batch-size", type=int, default=4) ap.add_argument("--data-root", type=str, default="") args = ap.parse_args() data_root = Path(args.data_root).resolve() if args.data_root else default_data_root() gscdb_root = Path(os.environ.get("GSCDB_ROOT", data_root / "GSCDB_gitclone" / "GSCDB")) os.environ.setdefault("GSCDB_ROOT", str(gscdb_root)) mol_folders = [p for p in (data_root / "GSCDB", data_root / "GSCDB_dmetals") if p.is_dir()] gscdb = utils.GSCDBDataset(gscdb_root=str(gscdb_root), mol_folders=mol_folders) keys = holdout_keys_from_bench(gscdb) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") out_dir = Path(args.out_dir) out_dir.mkdir(parents=True, exist_ok=True) print(f"holdout keys={len(keys)} device={device} data_root={data_root}", flush=True) mols_a, rxns_a, skip_a = build_cache_alpha(gscdb, keys) model_a = _load_ckpt(MACEAlphaHFReadout, args.alpha_ckpt, device) before_z = len(rxns_a) rxns_a = _filter_z(rxns_a, mols_a, frozenset(model_a.z_table.zs)) print(f"alpha cache={before_z} after_z={len(rxns_a)} skipped_fields={skip_a}", flush=True) err_a = _errors(forward_reaction_batch_alpha_hf, rxns_a, mols_a, model_a, device, args.batch_size) rows_a = [] for k in keys: nk = normalize_reaction_key(k) if nk not in err_a: continue ds = str(gscdb.df.loc[k, "Dataset"]) if k in gscdb.df.index else "" rows_a.append({"Reaction": k, "Dataset": ds, "Error_kcal": err_a[nk]}) pd.DataFrame(rows_a).to_csv(out_dir / "mace_alpha_errors.csv", index=False) mae_a = float(pd.Series([r["Error_kcal"] for r in rows_a]).abs().mean()) if rows_a else float("nan") print(f"alpha: n={len(rows_a)} MAE={mae_a:.6f} -> {out_dir / 'mace_alpha_errors.csv'}", flush=True) mols_d, rxns_d, skip_d = build_cache_delta(gscdb, keys) model_d = _load_ckpt(MACEDeltaB3LYPSkala, args.delta_ckpt, device) before_z = len(rxns_d) rxns_d = _filter_z(rxns_d, mols_d, frozenset(model_d.z_table.zs)) print(f"delta cache={before_z} after_z={len(rxns_d)} skipped_fields={skip_d}", flush=True) err_d = _errors(forward_reaction_batch_delta_b3lyp_skala, rxns_d, mols_d, model_d, device, args.batch_size) rows_d = [] for k in keys: nk = normalize_reaction_key(k) if nk not in err_d: continue ds = str(gscdb.df.loc[k, "Dataset"]) if k in gscdb.df.index else "" rows_d.append({"Reaction": k, "Dataset": ds, "Error_kcal": err_d[nk]}) pd.DataFrame(rows_d).to_csv(out_dir / "mace_delta_errors.csv", index=False) mae_d = float(pd.Series([r["Error_kcal"] for r in rows_d]).abs().mean()) if rows_d else float("nan") print(f"delta: n={len(rows_d)} MAE={mae_d:.6f} -> {out_dir / 'mace_delta_errors.csv'}", flush=True) if __name__ == "__main__": main()