#!/usr/bin/env python3 """Main-group Skala fine-tune.""" from __future__ import annotations import argparse import json import os import sys from dataclasses import dataclass from datetime import datetime from pathlib import Path import numpy as np import pandas as pd import torch from tqdm.auto import tqdm _CODE_TREE = Path(__file__).resolve().parents[1] sys.path.insert(0, str(_CODE_TREE)) from common.utils import ( # noqa: E402 CODE_ROOT, CONFIGS, DEFAULT_BATCH_SIZE, DEFAULT_CLIP_THRESHOLD, DEFAULT_EARLY_STOPPING_PATIENCE, DEFAULT_LR, DEFAULT_TOTAL_STEPS, DEFAULT_WARMUP_STEPS, HARTREE2KCAL, SKALA_FUNCTIONAL, DatasetWithBaseline, default_data_root, loss_weighted_mse, make_adafactor_cosine, make_name2baseline, name2baseline_indices, ) sys.path.insert(0, str(CODE_ROOT / "common")) import datasets as utils # noqa: E402 from skala.functional import load_functional # noqa: E402 ASSETS_DIR = CONFIGS / "skala_finetune_assets" DEFAULT_SPLIT_JSON = CONFIGS / "reaction_splits_skala_finetune.json" TM_GSCDB_PREFIXES = ("3d4dIPSS", "CUAGAU", "DAPD", "MME", "MOBH", "ROST", "TMD", "MOR", "TMB") def resolve_rdb7_dir(data_root: Path) -> Path: for name in ("RDB7_new", "RDB7"): p = data_root / name if p.is_dir(): return p raise FileNotFoundError(f"Neither RDB7_new nor RDB7 found under {data_root}") def resolve_gscdb_info_root(data_root: Path, override: str = "") -> Path: cands = [] if override: cands.append(Path(override)) env = os.environ.get("GSCDB_ROOT", "").strip() if env: cands.append(Path(env)) cands += [ data_root / "GSCDB_gitclone" / "GSCDB", data_root / "GSCDB", ] for c in cands: if (c / "Info" / "DatasetEval.csv").is_file(): return c raise FileNotFoundError( "GSCDB Info/DatasetEval.csv not found. Pass --gscdb-root or set GSCDB_ROOT. Tried: " + ", ".join(str(c) for c in cands) ) def is_tm_gscdb_reaction_name(name: str) -> bool: s = str(name) if s.startswith(("pts_", "rts_", "rp_")) or "_" not in s: return False sub = s.rsplit("_", 1)[0] return any(sub.startswith(p) for p in TM_GSCDB_PREFIXES) def filter_main_group_gscdb(names: list[str]) -> list[str]: return [n for n in names if not is_tm_gscdb_reaction_name(n)] def reinitialize_skala_weights(model: torch.nn.Module, seed: int | None = None) -> None: if seed is not None: torch.manual_seed(int(seed)) for p in model.parameters(): if p.dim() > 1: torch.nn.init.kaiming_uniform_(p, a=5**0.5) else: torch.nn.init.uniform_(p, -0.02, 0.02) @dataclass class SplitConfig: seed: int = 42 msr_val_size: int = 100 rdb7_val_size: int = 100 def get_fast_names(dataset): if hasattr(dataset, "mol_paths"): return [Path(p).stem for p in dataset.mol_paths] if hasattr(dataset, "paths"): return [Path(p).stem for p in dataset.paths] return [str(dataset[i].get("name", f"rxn_{i}")) for i in range(len(dataset))] def build_and_save_splits( cfg: SplitConfig, out_dir: Path, assets_dir: Path, data_root: Path, gscdb_root: Path ) -> dict: rng = np.random.default_rng(cfg.seed) gscdb_val_csv = assets_dir / "GSCDB_Slim300_errors_by_DFA.csv" gscdb_train_npy = assets_dir / "gscdb_training_part.npy" baseline_gscdb = assets_dir / "baseline_mae_gscdb.json" gscdb_baseline_names = set(json.load(open(baseline_gscdb)).keys()) # Slim300 = gscdb_val (val300 in Table 2) gscdb_val = [str(x) for x in pd.read_csv(gscdb_val_csv)["Reaction"].tolist()] val_names = set(gscdb_val) gscdb_train_raw = [str(x) for x in np.load(gscdb_train_npy, allow_pickle=True)] gscdb_train_raw = [x for x in gscdb_train_raw if x not in val_names] gscdb = utils.GSCDBDataset(gscdb_root=str(gscdb_root), mol_folder=str(data_root / "GSCDB")) available = set(gscdb.df.index.astype(str).tolist()) gscdb_train_full = sorted([x for x in gscdb_train_raw if x in available and x in gscdb_baseline_names]) target_train_size = len(gscdb_train_full) msr = utils.MSRReactionDataset( str(data_root / "MSR_TAE25" / "molecules"), str(data_root / "MSR_TAE25" / "atoms"), ) msr_names = sorted(get_fast_names(msr)) if len(msr_names) < cfg.msr_val_size + target_train_size: raise RuntimeError(f"Not enough MSR reactions: {len(msr_names)} < {cfg.msr_val_size + target_train_size}") rdb7 = utils.ReactionDataset([str(resolve_rdb7_dir(data_root))]) rdb7_names = sorted(get_fast_names(rdb7)) if len(rdb7_names) < cfg.rdb7_val_size + target_train_size: raise RuntimeError(f"Not enough RDB7 reactions: {len(rdb7_names)} < {cfg.rdb7_val_size + target_train_size}") gscdb_train_full = np.array(gscdb_train_full) g_perm = rng.permutation(len(gscdb_train_full)) gscdb_train = sorted(gscdb_train_full[g_perm[:target_train_size]].tolist()) msr_names = np.array(msr_names) msr_perm = rng.permutation(len(msr_names)) msr_val = sorted(msr_names[msr_perm[: cfg.msr_val_size]].tolist()) msr_train = sorted(msr_names[msr_perm[cfg.msr_val_size : cfg.msr_val_size + target_train_size]].tolist()) rdb7_names = np.array(rdb7_names) rdb7_perm = rng.permutation(len(rdb7_names)) rdb7_val = sorted(rdb7_names[rdb7_perm[: cfg.rdb7_val_size]].tolist()) rdb7_train = sorted(rdb7_names[rdb7_perm[cfg.rdb7_val_size : cfg.rdb7_val_size + target_train_size]].tolist()) payload = { "meta": {"seed": cfg.seed}, "splits": { "gscdb_train": gscdb_train, "gscdb_val": gscdb_val, "msr_train": msr_train, "msr_val": msr_val, "rdb7_train": rdb7_train, "rdb7_val": rdb7_val, }, } payload["sizes"] = {k: len(v) for k, v in payload["splits"].items()} split_dir = out_dir / "splits" split_dir.mkdir(parents=True, exist_ok=True) with open(split_dir / "reaction_splits_skala_finetune.json", "w") as f: json.dump(payload, f, indent=2) return payload def filter_dataset_by_names(dataset, names): names_set = set(names) if hasattr(dataset, "df") and hasattr(dataset, "keys"): keep = [k for k in dataset.keys if k in names_set] dataset.keys = keep dataset.df = dataset.df.loc[keep].copy() return dataset if hasattr(dataset, "mol_paths"): dataset.mol_paths = [p for p in dataset.mol_paths if Path(p).stem in names_set] return dataset if hasattr(dataset, "paths"): dataset.paths = [p for p in dataset.paths if Path(p).stem in names_set] return dataset raise RuntimeError("Unsupported dataset type for filtering") def load_or_build_splits(split_json: Path, out_dir: Path, assets_dir: Path, data_root: Path, gscdb_root: Path) -> dict: if split_json.is_file(): with open(split_json) as f: return json.load(f) print(f"[splits] building {split_json.name}", flush=True) return build_and_save_splits(SplitConfig(), out_dir, assets_dir, data_root, gscdb_root) def build_skala_features(mol, model, device: torch.device): rho = torch.as_tensor(mol["rho"], dtype=torch.float32, device=device) density = rho[:, 0, :] grad = rho[:, 1:4, :] kin = rho[:, 5, :] if rho.shape[1] > 5 else torch.zeros_like(density) if "atom_coords" not in mol: raise RuntimeError("molecule pickle missing atom_coords") atom_coords = torch.as_tensor(mol["atom_coords"], dtype=torch.float32, device=device) if atom_coords.ndim == 1: atom_coords = atom_coords.view(-1, 3) if atom_coords.numel() == 0 or atom_coords.shape[0] == 0 or atom_coords.shape[1] != 3: raise RuntimeError(f"Invalid atom_coords shape: {tuple(atom_coords.shape)}") if not torch.isfinite(atom_coords).all(): raise RuntimeError("atom_coords contains NaN/Inf values") feats = { "density": density, "grad": grad, "kin": kin, "grid_coords": torch.as_tensor(mol["grid_coords"], dtype=torch.float32, device=device), "grid_weights": torch.as_tensor(mol["weights"], dtype=torch.float32, device=device), "atom_coords": atom_coords, "coarse_0_atomic_coords": atom_coords, } required = list(getattr(model, "features", [])) missing = [k for k in required if k not in feats] if missing: raise RuntimeError(f"Missing required model features: {missing}") return {k: feats[k] for k in required} def forward_reaction_skala(reaction, model, device: torch.device): pred_rxn = torch.zeros((), dtype=torch.float32, device=device) ref_rxn = torch.as_tensor(float(reaction["energy"]) / HARTREE2KCAL, dtype=torch.float32, device=device) for _, mol in reaction["mols"].items(): coeff = float(mol.get("coeff", 1.0)) f = build_skala_features(mol, model, device) e_xc = model.get_exc(f) e_mol = ( torch.as_tensor(float(mol["E_unxc"]), dtype=torch.float32, device=device) + e_xc.to(torch.float32) + torch.as_tensor(float(mol.get("d3bj_b3lyp", 0.0)), dtype=torch.float32, device=device) ) pred_rxn = pred_rxn + coeff * e_mol return pred_rxn, ref_rxn @torch.no_grad() def eval_val_metrics(val_dataset, model, device: torch.device): model.eval() maes, losses = [], [] for i in tqdm(range(len(val_dataset)), desc="val"): rxn = val_dataset[i] pred_h, ref_h = forward_reaction_skala(rxn, model, device) pred_k = pred_h * HARTREE2KCAL ref_k = ref_h * HARTREE2KCAL b = torch.as_tensor(float(rxn["baseline_mae"]), dtype=pred_k.dtype, device=pred_k.device) losses.append(loss_weighted_mse(pred_k, ref_k, b).item()) maes.append(torch.abs(pred_h - ref_h).item() * HARTREE2KCAL) return ( float(np.mean(maes)) if maes else float("nan"), float(np.mean(losses)) if losses else float("nan"), ) def train( *, data_root: Path, gscdb_root: Path, assets_dir: Path, batch_size: int, total_steps: int, lr: float, warmup_steps: int, clip_threshold: float, seed: int, split_json: Path, out_dir: Path, checkpoint_every_epochs: int, train_from_scratch: bool = False, main_group_only: bool = False, gscdb_mol_folders: list[Path] | None = None, early_stopping_patience: int | None = DEFAULT_EARLY_STOPPING_PATIENCE, init_checkpoint: str | None = None, save_full_checkpoint: bool = False, ): device = torch.device("cuda" if torch.cuda.is_available() else "cpu") out_dir.mkdir(parents=True, exist_ok=True) ckpt_dir = out_dir / "checkpoints" ckpt_dir.mkdir(parents=True, exist_ok=True) utils.set_seed(seed) best_val = float("inf") stale = 0 payload = load_or_build_splits(split_json, out_dir, assets_dir, data_root, gscdb_root) splits = payload["splits"] if main_group_only: for key in ("gscdb_train", "gscdb_val"): before = len(splits[key]) splits[key] = filter_main_group_gscdb(list(splits[key])) print(f"{key}: {before} -> {len(splits[key])}") print("split sizes:", {k: len(v) for k, v in splits.items()}) mol_dirs = [str(p) for p in (gscdb_mol_folders or [data_root / "GSCDB"])] msr_mol, msr_atom = str(data_root / "MSR_TAE25" / "molecules"), str(data_root / "MSR_TAE25" / "atoms") rdb7 = str(resolve_rdb7_dir(data_root)) _g_train = utils.GSCDBDataset(gscdb_root=str(gscdb_root), mol_folders=mol_dirs) _g_val = utils.GSCDBDataset(gscdb_root=str(gscdb_root), mol_folders=mol_dirs) _msr_train = utils.MSRReactionDataset(msr_mol, msr_atom) _msr_val = utils.MSRReactionDataset(msr_mol, msr_atom) _rdb7_train = utils.ReactionDataset([rdb7]) _rdb7_val = utils.ReactionDataset([rdb7]) filter_dataset_by_names(_g_train, splits["gscdb_train"]) filter_dataset_by_names(_g_val, splits["gscdb_val"]) filter_dataset_by_names(_msr_train, splits["msr_train"]) filter_dataset_by_names(_msr_val, splits["msr_val"]) filter_dataset_by_names(_rdb7_train, splits["rdb7_train"]) filter_dataset_by_names(_rdb7_val, splits["rdb7_val"]) train_ds = utils.DatasetCompose([_g_train, _msr_train, _rdb7_train]) val_ds = utils.DatasetCompose([_g_val, _msr_val, _rdb7_val]) subset2baseline, msr_global, rdb7_global = make_name2baseline(assets_dir) idx_tr = name2baseline_indices(train_ds, subset2baseline, msr_global, rdb7_global) idx_va = name2baseline_indices(val_ds, subset2baseline, msr_global, rdb7_global) train_dataset_b97 = DatasetWithBaseline(train_ds, idx_tr) val_dataset_b97 = DatasetWithBaseline(val_ds, idx_va) print(f"train/val = {len(train_dataset_b97)}/{len(val_dataset_b97)} device={device}", flush=True) model = load_functional(SKALA_FUNCTIONAL, device=device).to(device) if train_from_scratch: reinitialize_skala_weights(model, seed=seed) if init_checkpoint: ckpt = torch.load(init_checkpoint, map_location=device, weights_only=False) state = ckpt["state_dict"] if isinstance(ckpt, dict) and "state_dict" in ckpt else ckpt missing, unexpected = model.load_state_dict(state, strict=False) print( f"[init] Skala 1.0: loaded {init_checkpoint} " f"(missing={len(missing)} unexpected={len(unexpected)})", flush=True, ) if missing or unexpected: raise RuntimeError( f"checkpoint mismatch: missing={missing[:5]} unexpected={unexpected[:5]}" ) model.train() optimizer, scheduler = make_adafactor_cosine( model, max_lr=lr, warmup_steps=warmup_steps, total_steps=total_steps, clip_threshold=clip_threshold ) history = [] global_step = 0 epoch = 0 in_batch = 0 run_meta = { "batch_size": batch_size, "total_steps": total_steps, "lr": lr, "warmup_steps": warmup_steps, "clip_threshold": clip_threshold, "seed": seed, "device": str(device), "init_checkpoint": init_checkpoint, } with open(out_dir / "run_meta.json", "w") as f: json.dump(run_meta, f, indent=2) pbar = tqdm(total=total_steps, desc="train steps") while global_step < total_steps: model.train() train_dataset_b97.shuffle(seed=seed + epoch) optimizer.zero_grad(set_to_none=True) in_batch = 0 train_mae_accum = [] train_loss_wmse_accum = [] for i in range(len(train_dataset_b97)): rxn = train_dataset_b97[i] pred_h, ref_h = forward_reaction_skala(rxn, model, device) pred_k = pred_h * HARTREE2KCAL ref_k = ref_h * HARTREE2KCAL b = torch.as_tensor(float(rxn["baseline_mae"]), dtype=pred_k.dtype, device=pred_k.device) lw = loss_weighted_mse(pred_k, ref_k, b) train_loss_wmse_accum.append(lw.detach().item()) loss = lw / batch_size loss.backward() train_mae_accum.append(torch.abs(pred_h - ref_h).item() * HARTREE2KCAL) in_batch += 1 if in_batch == batch_size: optimizer.step() scheduler.step() optimizer.zero_grad(set_to_none=True) in_batch = 0 global_step += 1 pbar.update(1) if global_step >= total_steps: break if in_batch > 0 and global_step < total_steps: optimizer.step() scheduler.step() optimizer.zero_grad(set_to_none=True) global_step += 1 pbar.update(1) val_mae, val_loss_wmse = eval_val_metrics(val_dataset_b97, model, device) row = { "epoch": epoch, "global_step": global_step, "train_loss_weighted_mse": float(np.mean(train_loss_wmse_accum)) if train_loss_wmse_accum else float("nan"), "val_loss_weighted_mse": val_loss_wmse, "train_mae_kcal": float(np.mean(train_mae_accum)) if train_mae_accum else float("nan"), "val_mae_kcal": val_mae, "lr": float(optimizer.param_groups[0]["lr"]), } history.append(row) print(row, flush=True) with open(out_dir / "history.json", "w") as f: json.dump(history, f, indent=2) if val_loss_wmse < best_val: best_val = val_loss_wmse stale = 0 torch.save(model.state_dict(), ckpt_dir / "best.pt") else: stale += 1 if early_stopping_patience is not None and stale >= early_stopping_patience: print(f"early stop at epoch {epoch}", flush=True) break epoch += 1 if checkpoint_every_epochs > 0 and epoch % checkpoint_every_epochs == 0: torch.save(model.state_dict(), ckpt_dir / f"epoch_{epoch:04d}_step_{global_step}.pt") if global_step >= total_steps: break pbar.close() torch.save(model.state_dict(), ckpt_dir / "final.pt") if save_full_checkpoint: torch.save( { "model": model.state_dict(), "optimizer": optimizer.state_dict(), "scheduler": scheduler.state_dict(), "global_step": global_step, "epoch": epoch, "seed": seed, }, ckpt_dir / "final_full.pt", ) print(f"[checkpoint] saved full optim/sched state → {ckpt_dir / 'final_full.pt'}", flush=True) print("Done. Checkpoints:", ckpt_dir, flush=True) def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--batch-size", type=int, default=DEFAULT_BATCH_SIZE) ap.add_argument("--total-steps", type=int, default=DEFAULT_TOTAL_STEPS) ap.add_argument("--lr", type=float, default=DEFAULT_LR) ap.add_argument("--warmup-steps", type=int, default=DEFAULT_WARMUP_STEPS) ap.add_argument("--clip-threshold", type=float, default=DEFAULT_CLIP_THRESHOLD) ap.add_argument("--seed", type=int, default=42) ap.add_argument( "--early-stopping-patience", type=int, default=DEFAULT_EARLY_STOPPING_PATIENCE, help="epochs without val improvement before stop; 0 disables", ) ap.add_argument("--data-root", type=str, default=str(default_data_root())) ap.add_argument("--gscdb-root", type=str, default="") ap.add_argument("--assets-dir", type=str, default=str(ASSETS_DIR)) ap.add_argument("--split-json", type=str, default=str(DEFAULT_SPLIT_JSON)) ap.add_argument("--out-dir", type=str, default="") ap.add_argument("--checkpoint-every-epochs", type=int, default=10, help="Save model.state_dict every N completed epochs (0 = disable; final.pt always saved).") ap.add_argument("--train-from-scratch", action="store_true") ap.add_argument("--init-checkpoint", type=str, default="", help="Optional state_dict (.pt) to load after Skala init / before training.") ap.add_argument("--save-full-checkpoint", action="store_true", help="Also write final_full.pt with model+optimizer+scheduler state.") ap.add_argument("--main-group-only", action="store_true", help="Drop transition-metal GSCDB subset reactions from gscdb_train/gscdb_val.") ap.add_argument("--gscdb-mol-folders", type=str, default="GSCDB,GSCDB_dmetals", help="Comma-separated folder names under --data-root; union for availability, " "first existing folder wins when loading each pickle.") args = ap.parse_args(argv) data_root = Path(args.data_root).resolve() gscdb_root = resolve_gscdb_info_root(data_root, args.gscdb_root) if args.out_dir: out_dir = Path(args.out_dir) else: sub = "skala_train_from_scratch" if args.train_from_scratch else "skala_finetune" out_dir = CODE_ROOT / "runs" / sub / datetime.now().strftime("%Y%m%d_%H%M%S") warmup = min(args.warmup_steps, max(0, args.total_steps - 1)) folder_names = [p.strip() for p in args.gscdb_mol_folders.split(",") if p.strip()] gscdb_mol_folders = [data_root / p for p in folder_names if (data_root / p).is_dir()] if not gscdb_mol_folders: raise FileNotFoundError(f"None of the GSCDB mol folders {folder_names} exist under {data_root}") train( data_root=data_root, gscdb_root=gscdb_root, assets_dir=Path(args.assets_dir), batch_size=max(1, args.batch_size), total_steps=args.total_steps, lr=args.lr, warmup_steps=warmup, clip_threshold=args.clip_threshold, seed=args.seed, split_json=Path(args.split_json), out_dir=out_dir, checkpoint_every_epochs=max(0, args.checkpoint_every_epochs), train_from_scratch=bool(args.train_from_scratch), main_group_only=bool(args.main_group_only), gscdb_mol_folders=gscdb_mol_folders, early_stopping_patience=(args.early_stopping_patience or None), init_checkpoint=(args.init_checkpoint or None), save_full_checkpoint=bool(args.save_full_checkpoint), ) if __name__ == "__main__": main()