#!/usr/bin/env python3 """TM Skala fine-tune.""" from __future__ import annotations import argparse import json import os import sys from datetime import datetime from pathlib import Path import numpy as np import pandas as pd import torch from tqdm.auto import tqdm _CODE_TREE = Path(__file__).resolve().parents[1] sys.path.insert(0, str(_CODE_TREE)) from common.utils import ( CODE_ROOT, CONFIGS, DEFAULT_BATCH_SIZE, DEFAULT_CLIP_THRESHOLD, DEFAULT_EARLY_STOPPING_PATIENCE, DEFAULT_LR, DEFAULT_TOTAL_STEPS, DEFAULT_WARMUP_STEPS, HARTREE2KCAL, SKALA_FUNCTIONAL, default_data_root, loss_weighted_mse, make_adafactor_cosine, ) sys.path.insert(0, str(CODE_ROOT / "common")) import datasets as utils # noqa: E402 from skala.functional import load_functional # noqa: E402 def resolve_gscdb_info_root(data_root: Path, override: str = "") -> Path: if override: p = Path(override) if not (p / "Info" / "DatasetEval.csv").is_file(): raise FileNotFoundError(p) return p env = os.environ.get("GSCDB_ROOT", "").strip() cands = [] if env: cands.append(Path(env)) cands += [ data_root / "GSCDB_gitclone" / "GSCDB", ] for p in cands: if (p / "Info" / "DatasetEval.csv").is_file(): return p raise FileNotFoundError("GSCDB Info/DatasetEval.csv not found; set --gscdb-root or GSCDB_ROOT") def build_skala_features(mol, model, device): rho = torch.as_tensor(mol["rho"], dtype=torch.float32, device=device) density = rho[:, 0, :] grad = rho[:, 1:4, :] kin = rho[:, 5, :] if rho.shape[1] > 5 else torch.zeros_like(density) atom_coords = torch.as_tensor(mol["atom_coords"], dtype=torch.float32, device=device) if atom_coords.ndim == 1: atom_coords = atom_coords.view(-1, 3) feats = { "density": density, "grad": grad, "kin": kin, "grid_coords": torch.as_tensor(mol["grid_coords"], dtype=torch.float32, device=device), "grid_weights": torch.as_tensor(mol["weights"], dtype=torch.float32, device=device), "atom_coords": atom_coords, "coarse_0_atomic_coords": atom_coords, } return {k: feats[k] for k in getattr(model, "features", feats)} def forward_reaction(reaction, model, required_features, device): pred = torch.zeros((), dtype=torch.float32, device=device) ref = torch.as_tensor(float(reaction["energy"]) / HARTREE2KCAL, dtype=torch.float32, device=device) for _, mol in reaction["mols"].items(): coeff = float(mol.get("coeff", 1.0)) f = build_skala_features(mol, model, device) f = {k: f[k] for k in required_features} e_xc = model.get_exc(f) e_mol = ( torch.as_tensor(float(mol["E_unxc"]), dtype=torch.float32, device=device) + e_xc.to(torch.float32) + torch.as_tensor(float(mol.get("d3bj_b3lyp", 0.0)), dtype=torch.float32, device=device) ) pred = pred + coeff * e_mol return pred, ref class ReactionDatasetWithBaseline: def __init__(self, base, names, baselines): name_to_i = {} for i in range(len(base)): name_to_i[str(base[i]["name"])] = i self.base = base self.indices = [] self.baselines = [] for name, b in zip(names, baselines): if name not in name_to_i: continue self.indices.append(name_to_i[name]) self.baselines.append(float(b)) self.indices = np.asarray(self.indices, dtype=int) self.baselines = np.asarray(self.baselines, dtype=np.float64) def __len__(self): return len(self.indices) def __getitem__(self, i): rxn = dict(self.base[int(self.indices[i])]) rxn["baseline_mae"] = float(self.baselines[i]) return rxn def shuffle(self, seed=None): rng = np.random.default_rng(seed) perm = rng.permutation(len(self.indices)) self.indices = self.indices[perm] self.baselines = self.baselines[perm] @torch.no_grad() def eval_dataset(ds, model, required_features, device): model.eval() maes, losses = [], [] for i in range(len(ds)): rxn = ds[i] pred_h, ref_h = forward_reaction(rxn, model, required_features, device) pred_k = pred_h * HARTREE2KCAL ref_k = ref_h * HARTREE2KCAL b = torch.as_tensor(float(rxn["baseline_mae"]), dtype=pred_k.dtype, device=device) losses.append(loss_weighted_mse(pred_k, ref_k, b).item()) maes.append(torch.abs(pred_h - ref_h).item() * HARTREE2KCAL) return ( float(np.mean(maes)) if maes else float("nan"), float(np.mean(losses)) if losses else float("nan"), ) def load_split_csv(path: Path): df = pd.read_csv(path) return df["name"].astype(str).tolist(), df["baseline_mae"].astype(float).tolist() def fine_tune_tm( train_ds, val_ds, test_ds, *, out_dir: Path, total_steps: int = DEFAULT_TOTAL_STEPS, batch_size: int = 10, lr: float = DEFAULT_LR, warmup_steps: int = DEFAULT_WARMUP_STEPS, clip_threshold: float = DEFAULT_CLIP_THRESHOLD, seed: int = 42, early_stopping_patience: int | None = DEFAULT_EARLY_STOPPING_PATIENCE, device: torch.device | None = None, ): device = device or torch.device("cuda" if torch.cuda.is_available() else "cpu") out_dir.mkdir(parents=True, exist_ok=True) utils.set_seed(seed) model = load_functional(SKALA_FUNCTIONAL, device=device).to(device) required = list(getattr(model, "features", [])) warmup_steps = min(warmup_steps, max(0, total_steps - 1)) optimizer, scheduler = make_adafactor_cosine( model, max_lr=lr, warmup_steps=warmup_steps, total_steps=total_steps, clip_threshold=clip_threshold ) history = [] global_step = 0 epoch = 0 best_val = float("inf") stale = 0 meta = { "batch_size": batch_size, "total_steps": total_steps, "lr": lr, "warmup_steps": warmup_steps, "clip_threshold": clip_threshold, "seed": seed, "early_stopping_patience": early_stopping_patience, "n_train": len(train_ds), "n_val": len(val_ds), "n_test": len(test_ds), } (out_dir / "run_meta.json").write_text(json.dumps(meta, indent=2)) pbar = tqdm(total=total_steps, desc="tm steps") while global_step < total_steps: model.train() train_ds.shuffle(seed=seed + epoch) optimizer.zero_grad(set_to_none=True) in_batch = 0 train_mae = [] for i in range(len(train_ds)): rxn = train_ds[i] pred_h, ref_h = forward_reaction(rxn, model, required, device) pred_k = pred_h * HARTREE2KCAL ref_k = ref_h * HARTREE2KCAL b = torch.as_tensor(float(rxn["baseline_mae"]), dtype=pred_k.dtype, device=device) loss = loss_weighted_mse(pred_k, ref_k, b) / batch_size loss.backward() train_mae.append(torch.abs(pred_h - ref_h).item() * HARTREE2KCAL) in_batch += 1 if in_batch == batch_size: optimizer.step() scheduler.step() optimizer.zero_grad(set_to_none=True) in_batch = 0 global_step += 1 pbar.update(1) if global_step >= total_steps: break if in_batch > 0 and global_step < total_steps: optimizer.step() scheduler.step() optimizer.zero_grad(set_to_none=True) global_step += 1 pbar.update(1) val_mae, val_loss = eval_dataset(val_ds, model, required, device) test_mae, test_loss = eval_dataset(test_ds, model, required, device) row = { "epoch": epoch, "global_step": global_step, "train_mae_kcal": float(np.mean(train_mae)) if train_mae else float("nan"), "val_mae_kcal": val_mae, "val_loss_weighted_mse": val_loss, "holdout_mae_kcal": test_mae, "holdout_loss_weighted_mse": test_loss, "lr": float(optimizer.param_groups[0]["lr"]), } history.append(row) print(row, flush=True) (out_dir / "tm_finetune_history.json").write_text(json.dumps(history, indent=2)) torch.save(model.state_dict(), out_dir / f"tm_finetune_step_{global_step}.pt") if val_loss < best_val: best_val = val_loss stale = 0 torch.save(model.state_dict(), out_dir / "tm_finetune_best.pt") else: stale += 1 if early_stopping_patience is not None and stale >= early_stopping_patience: print(f"early stop at epoch {epoch}", flush=True) break epoch += 1 if global_step >= total_steps: break pbar.close() torch.save(model.state_dict(), out_dir / "tm_finetune_final.pt") return history def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--lr", type=float, default=DEFAULT_LR) ap.add_argument("--total-steps", type=int, default=DEFAULT_TOTAL_STEPS) ap.add_argument("--warmup-steps", type=int, default=DEFAULT_WARMUP_STEPS) ap.add_argument("--batch-size", type=int, default=DEFAULT_BATCH_SIZE) ap.add_argument("--clip-threshold", type=float, default=DEFAULT_CLIP_THRESHOLD) ap.add_argument("--seed", type=int, default=42) ap.add_argument( "--early-stopping-patience", type=int, default=DEFAULT_EARLY_STOPPING_PATIENCE, help="epochs without val improvement before stop; 0 disables", ) ap.add_argument("--data-root", type=str, default=str(default_data_root())) ap.add_argument("--tm-mol-folder", type=str, default="") ap.add_argument("--gscdb-root", type=str, default="") ap.add_argument("--splits-dir", type=str, default=str(CONFIGS)) ap.add_argument("--out-dir", type=str, default="") ap.add_argument("--device", type=str, default="cuda" if torch.cuda.is_available() else "cpu") args = ap.parse_args(argv) device = torch.device(args.device) data_root = Path(args.data_root).resolve() tm_folder = Path(args.tm_mol_folder) if args.tm_mol_folder else data_root / "GSCDB_dmetals" gscdb_root = resolve_gscdb_info_root(data_root, args.gscdb_root) splits = Path(args.splits_dir) out_dir = Path(args.out_dir) if args.out_dir else ( CODE_ROOT / "runs" / "skala_tm" / datetime.now().strftime("%Y%m%d_%H%M%S") ) base = utils.GSCDBDataset(gscdb_root=str(gscdb_root), mol_folder=str(tm_folder)) tr_n, tr_b = load_split_csv(splits / "tm_train_split.csv") va_n, va_b = load_split_csv(splits / "tm_val_split.csv") te_n, te_b = load_split_csv(splits / "tm_test_split.csv") train_ds = ReactionDatasetWithBaseline(base, tr_n, tr_b) val_ds = ReactionDatasetWithBaseline(base, va_n, va_b) test_ds = ReactionDatasetWithBaseline(base, te_n, te_b) print(f"train/val/test = {len(train_ds)}/{len(val_ds)}/{len(test_ds)}", flush=True) fine_tune_tm( train_ds, val_ds, test_ds, out_dir=out_dir, total_steps=args.total_steps, batch_size=max(1, args.batch_size), lr=args.lr, warmup_steps=args.warmup_steps, clip_threshold=args.clip_threshold, seed=args.seed, early_stopping_patience=(args.early_stopping_patience or None), device=device, ) if __name__ == "__main__": main()