#!/usr/bin/env python3 import csv import json import math import os import random from collections import defaultdict from statistics import mean import numpy as np # Default GSCDB metadata root; override with GSCDB_ROOT. _CODE = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) _DEFAULT_GSCDB = os.environ.get("GSCDB_ROOT", "").strip() if not _DEFAULT_GSCDB: for cand in ( os.path.join(_CODE, "data_pickles", "GSCDB_gitclone", "GSCDB"), os.path.join(_CODE, "data_pickles", "GSCDB"), ): if os.path.isfile(os.path.join(cand, "Info", "DatasetEval.csv")): _DEFAULT_GSCDB = cand break GSCDB_ROOT = _DEFAULT_GSCDB OUT_DIR = os.environ.get( "SAMPLING_OUT_DIR", os.path.join(_CODE, "configs"), ) EXCLUDE_DATASETS = {"Dip146", "Pol130", "HR46", "T144", "V30", "SC74", "OEEFD"} N_SAMPLES = 300 N_SEEDS = 20 BASELINE_DFA = "PBE0" def read_csv(path): with open(path, newline="") as f: return list(csv.DictReader(f)) def safe_float(x, default=0.0): try: return float(x) except Exception: return default def load_data(): info_eval = read_csv(os.path.join(GSCDB_ROOT, "Info", "DatasetEval.csv")) info_datasets = read_csv(os.path.join(GSCDB_ROOT, "Info", "Datasets.csv")) std_errors = read_csv(os.path.join(GSCDB_ROOT, "Info", "Standard_errors.csv")) results_errors = read_csv(os.path.join(GSCDB_ROOT, "Analysis", "Results_errors.csv")) slim300 = read_csv(os.path.join(GSCDB_ROOT, "Analysis", "GSCDB_Slim300.csv")) dataset_to_cat = {} for r in info_datasets: name = r.get("Name", "") cat = r.get("Datatype_Short", "UNK") if name: dataset_to_cat[name] = cat sigma_by_dataset = {} for r in std_errors: ds = r.get("Dataset", "") metric = safe_float(r.get("Metric", ""), default=np.nan) if ds and not math.isnan(metric) and metric != 0.0: sigma_by_dataset[ds] = metric # all DFAs from Results_errors fixed_cols = {"Reaction", "Dataset", "Reference"} dfa_cols = [c for c in results_errors[0].keys() if c not in fixed_cols] # candidate pool: energy datasets only, with errors available eval_by_reaction = {} for r in info_eval: ds = r.get("Dataset", "") rid = r.get("Reaction", "") if not rid or ds in EXCLUDE_DATASETS: continue eval_by_reaction[rid] = { "Reaction": rid, "Dataset": ds, "Category": dataset_to_cat.get(ds, "UNK"), } err_by_reaction = {} for r in results_errors: rid = r.get("Reaction", "") if rid in eval_by_reaction: ds = r.get("Dataset", "") if ds in EXCLUDE_DATASETS: continue row = { "Reaction": rid, "Dataset": ds, "Category": eval_by_reaction[rid]["Category"], } for dfa in dfa_cols: row[dfa] = safe_float(r.get(dfa, "0")) err_by_reaction[rid] = row pool = list(err_by_reaction.values()) # optimized reference subset (from provided slim300 list) slim_ids = {r["Reaction"] for r in slim300 if r.get("Reaction")} optimized_subset = [err_by_reaction[rid] for rid in slim_ids if rid in err_by_reaction] return pool, optimized_subset, sigma_by_dataset, dfa_cols def mae_for_subset(rows, dfa): return mean(abs(r[dfa]) for r in rows) if rows else float("nan") def ner_for_subset(rows, dfa, sigma_by_dataset): by_dataset = defaultdict(list) for r in rows: by_dataset[r["Dataset"]].append(abs(r[dfa])) vals = [] for ds, errs in by_dataset.items(): sigma = sigma_by_dataset.get(ds) if sigma is None or sigma == 0: continue vals.append((sum(errs) / len(errs)) / sigma) return sum(vals) / len(vals) if vals else float("nan") def metrics(rows, dfa_cols, sigma_by_dataset): out = {} for dfa in dfa_cols: out[dfa] = { "MAE": mae_for_subset(rows, dfa), "NER": ner_for_subset(rows, dfa, sigma_by_dataset), } return out def compare(full_m, sub_m, dfa_cols): rel_mae = [] rel_ner = [] rel_ner_each = [] for dfa in dfa_cols: mae_full = full_m[dfa]["MAE"] mae_sub = sub_m[dfa]["MAE"] ner_full = full_m[dfa]["NER"] ner_sub = sub_m[dfa]["NER"] if mae_full and not math.isnan(mae_full): rel_mae.append(abs(mae_sub - mae_full) / mae_full) if ner_full and not math.isnan(ner_full): v = abs(ner_sub - ner_full) / ner_full rel_ner.append(v) rel_ner_each.append(v) delta_mad = float(np.mean(rel_mae)) if rel_mae else float("nan") delta_ner = float(np.mean(rel_ner)) if rel_ner else float("nan") delta_max = float(np.max(rel_ner_each)) if rel_ner_each else float("nan") score = delta_mad + delta_ner + 5.0 * delta_max return { "Delta_MAD": delta_mad, "Delta_NER": delta_ner, "Delta_max": delta_max, "F_score": score, } def sample_random(pool, n, rng): idx = rng.sample(range(len(pool)), n) return [pool[i] for i in idx] def sample_stratified(pool, n, key, rng): groups = defaultdict(list) for i, row in enumerate(pool): groups[row[key]].append(i) total = len(pool) alloc = {} for g, ids in groups.items(): alloc[g] = int(round(n * len(ids) / total)) # fix rounding drift diff = n - sum(alloc.values()) order = sorted(groups.keys(), key=lambda g: len(groups[g]), reverse=True) j = 0 while diff != 0 and order: g = order[j % len(order)] if diff > 0: alloc[g] += 1 diff -= 1 else: if alloc[g] > 0: alloc[g] -= 1 diff += 1 j += 1 chosen = [] for g, ids in groups.items(): k = min(alloc[g], len(ids)) chosen.extend(rng.sample(ids, k)) if len(chosen) > n: chosen = rng.sample(chosen, n) elif len(chosen) < n: remain = list(set(range(len(pool))) - set(chosen)) chosen.extend(rng.sample(remain, n - len(chosen))) return [pool[i] for i in chosen] def sample_error_binned(pool, n, rng, dfa=BASELINE_DFA): vals = np.array([abs(r[dfa]) for r in pool]) q = np.quantile(vals, [0, 0.25, 0.5, 0.75, 1.0]) bins = defaultdict(list) for i, v in enumerate(vals): b = int(np.searchsorted(q, v, side="right") - 1) b = min(max(b, 0), 3) bins[b].append(i) # equal allocation across quartiles per = n // 4 chosen = [] for b in range(4): ids = bins[b] k = min(per, len(ids)) chosen.extend(rng.sample(ids, k)) while len(chosen) < n: remain = list(set(range(len(pool))) - set(chosen)) chosen.append(rng.choice(remain)) return [pool[i] for i in chosen[:n]] def run(): os.makedirs(OUT_DIR, exist_ok=True) pool, optimized, sigma_by_dataset, dfa_cols = load_data() if len(optimized) == 0: raise RuntimeError("Optimized subset is empty; check GSCDB_Slim300.csv") full_metrics = metrics(pool, dfa_cols, sigma_by_dataset) optimized_metrics = metrics(optimized, dfa_cols, sigma_by_dataset) optimized_cmp = compare(full_metrics, optimized_metrics, dfa_cols) strategies = { "random_uniform": lambda rng: sample_random(pool, N_SAMPLES, rng), "stratified_category": lambda rng: sample_stratified(pool, N_SAMPLES, "Category", rng), "stratified_dataset": lambda rng: sample_stratified(pool, N_SAMPLES, "Dataset", rng), "error_binned_pbe0": lambda rng: sample_error_binned(pool, N_SAMPLES, rng, dfa=BASELINE_DFA), } rows = [] summary = [] for name, sampler in strategies.items(): vals = {"Delta_MAD": [], "Delta_NER": [], "Delta_max": [], "F_score": []} for seed in range(N_SEEDS): rng = random.Random(seed) subset = sampler(rng) sub_metrics = metrics(subset, dfa_cols, sigma_by_dataset) cmp = compare(full_metrics, sub_metrics, dfa_cols) for k in vals: vals[k].append(cmp[k]) rows.append({ "strategy": name, "seed": seed, **cmp, }) summary.append({ "strategy": name, "n_seeds": N_SEEDS, "Delta_MAD_mean": float(np.mean(vals["Delta_MAD"])), "Delta_MAD_std": float(np.std(vals["Delta_MAD"])), "Delta_NER_mean": float(np.mean(vals["Delta_NER"])), "Delta_NER_std": float(np.std(vals["Delta_NER"])), "Delta_max_mean": float(np.mean(vals["Delta_max"])), "Delta_max_std": float(np.std(vals["Delta_max"])), "F_score_mean": float(np.mean(vals["F_score"])), "F_score_std": float(np.std(vals["F_score"])), }) # include optimized reference summary.append({ "strategy": "optimized_reference_slim300", "n_seeds": 1, "Delta_MAD_mean": optimized_cmp["Delta_MAD"], "Delta_MAD_std": 0.0, "Delta_NER_mean": optimized_cmp["Delta_NER"], "Delta_NER_std": 0.0, "Delta_max_mean": optimized_cmp["Delta_max"], "Delta_max_std": 0.0, "F_score_mean": optimized_cmp["F_score"], "F_score_std": 0.0, }) # write per-seed csv per_seed_path = os.path.join(OUT_DIR, "sampling_ablation_per_seed.csv") with open(per_seed_path, "w", newline="") as f: w = csv.DictWriter(f, fieldnames=["strategy", "seed", "Delta_MAD", "Delta_NER", "Delta_max", "F_score"]) w.writeheader() for r in rows: w.writerow(r) summary_path = os.path.join(OUT_DIR, "sampling_ablation_summary.csv") with open(summary_path, "w", newline="") as f: cols = [ "strategy", "n_seeds", "Delta_MAD_mean", "Delta_MAD_std", "Delta_NER_mean", "Delta_NER_std", "Delta_max_mean", "Delta_max_std", "F_score_mean", "F_score_std", ] w = csv.DictWriter(f, fieldnames=cols) w.writeheader() for r in summary: w.writerow(r) metadata = { "pool_size": len(pool), "subset_size": N_SAMPLES, "optimized_subset_size": len(optimized), "n_dfas": len(dfa_cols), "n_seeds": N_SEEDS, "exclude_datasets": sorted(EXCLUDE_DATASETS), "baseline_dfa_for_error_binning": BASELINE_DFA, } with open(os.path.join(OUT_DIR, "sampling_ablation_metadata.json"), "w") as f: json.dump(metadata, f, indent=2) print("Wrote:", per_seed_path) print("Wrote:", summary_path) print("Wrote:", os.path.join(OUT_DIR, "sampling_ablation_metadata.json")) if __name__ == "__main__": run()