"""Reaction datasets used by the trainers (GSCDB / MSR / RDB7 pickles).""" from __future__ import annotations import json import os import pickle import random from collections import Counter import numpy as np import pandas as pd import torch hartree2kcal = 627.509474 def set_seed(seed: int = 42): random.seed(seed) os.environ["PYTHONHASHSEED"] = str(seed) np.random.seed(seed) torch.manual_seed(seed) if torch.cuda.is_available(): torch.cuda.manual_seed(seed) class ReactionDataset: def __init__(self, paths2subsets): self.paths = [] for subset in paths2subsets: for reaction in os.listdir(subset): if reaction.endswith(".pickle"): self.paths.append(os.path.join(subset, reaction)) def __len__(self): return len(self.paths) def __getitem__(self, i): with open(self.paths[i], "rb") as handle: return pickle.load(handle) class DatasetCompose: def __init__(self, datasets): self.datasets = datasets self.index_map = [] for d_idx, dataset in enumerate(datasets): for i in range(len(dataset)): self.index_map.append((d_idx, i)) self.index_map = np.array(self.index_map, dtype=object) def __len__(self): return len(self.index_map) def __getitem__(self, idx): d_idx, local_idx = self.index_map[idx] return self.datasets[d_idx][local_idx] def shuffle(self, seed=None): rng = np.random.default_rng(seed) rng.shuffle(self.index_map) class MSRReactionDataset: def __init__(self, mol_folder, atom_folder): self.atoms = {} for f in os.listdir(atom_folder): if f.endswith(".pickle"): name = os.path.splitext(f)[0] with open(os.path.join(atom_folder, f), "rb") as handle: self.atoms[name] = pickle.load(handle) self.mol_paths = [ os.path.join(mol_folder, f) for f in os.listdir(mol_folder) if f.endswith(".pickle") ] def __len__(self): return len(self.mol_paths) def __getitem__(self, i): with open(self.mol_paths[i], "rb") as handle: mol_data = pickle.load(handle) name = os.path.splitext(os.path.basename(self.mol_paths[i]))[0] if "symbols" not in mol_data: mol_data["symbols"] = mol_data["atoms"] if "energy" not in mol_data: msr_root = os.environ.get("MSR_ACC_ROOT", "").strip() if not msr_root: raise KeyError(f"no energy in {name}; set MSR_ACC_ROOT or bake it into the pickle") with open(os.path.join(msr_root, "tae25", f"{name}.json")) as file: mol_data["energy"] = json.load(file)["extras"]["tae@w1-f12"] * hartree2kcal reaction = {"mols": {}, "name": name, "energy": mol_data["energy"]} mol_entry = mol_data mol_entry["coeff"] = -1 reaction["mols"][name] = mol_entry for symbol, count in Counter(mol_data["symbols"]).items(): atom_copy = self.atoms[symbol].copy() atom_copy["coeff"] = count reaction["mols"][symbol] = atom_copy return reaction class GSCDBDataset: def __init__(self, gscdb_root, mol_folder=None, mol_folders=None, suffix=".pickle"): if mol_folders is not None: self.mol_folders = list(mol_folders) elif mol_folder is not None: self.mol_folders = [mol_folder] else: raise ValueError("pass mol_folder= or mol_folders=") self.mol_folder = self.mol_folders[0] self.suffix = suffix df = pd.read_csv(os.path.join(gscdb_root, "Info", "DatasetEval.csv"), index_col=0) exclude = {"Dip146", "Pol130", "HR46", "T144", "V30", "SC74", "OEEFD"} df = df[~df["Dataset"].isin(exclude)].copy() df["Reference"] *= 627.50947406 available = set() for folder in self.mol_folders: if not os.path.isdir(folder): continue for f in os.listdir(folder): if f.endswith(suffix): available.add(f[: -len(suffix)]) keep = [] for idx, row in df.iterrows(): parts = row["Stoichiometry"].split(",") ok = all(parts[i].strip() in available for i in range(1, len(parts), 2)) if ok: keep.append(idx) self.df = df.loc[keep].copy() self.keys = self.df.index.tolist() print(f"GSCDB: {len(self.keys)} reactions", flush=True) def shuffle(self, seed=None): rng = np.random.default_rng(seed) keys = list(self.keys) rng.shuffle(keys) self.keys = keys def _resolve_mol_path(self, mol_name): for folder in self.mol_folders: path = os.path.join(folder, f"{mol_name}{self.suffix}") if os.path.isfile(path): return path return os.path.join(self.mol_folders[0], f"{mol_name}{self.suffix}") def __len__(self): return len(self.keys) def __getitem__(self, idx): rid = self.keys[idx] if isinstance(idx, int) else idx row = self.df.loc[rid] parts = row["Stoichiometry"].split(",") mols = {} for i in range(len(parts) // 2): coeff = float(parts[2 * i]) mol_name = parts[2 * i + 1].strip() with open(self._resolve_mol_path(mol_name), "rb") as f: mol_data = pickle.load(f) mol_data["coeff"] = coeff mols[mol_name] = mol_data return { "name": rid, "dataset": row["Dataset"], "energy": float(row["Reference"]), "mols": mols, }