#!/usr/bin/env python3 """Slim validation subset via genetic search.""" from __future__ import annotations import argparse import csv import json import math import random import sys from collections import defaultdict from pathlib import Path import numpy as np CODE_ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(CODE_ROOT)) from validation_subset.sampling_ablation import ( # noqa: E402 compare, load_data, metrics, sample_stratified, ) POP_SIZE = 60 N_GENS = 400 ELITISM = 10 MUTATION_RATE = 0.12 EARLY_STOP = 100 N_STARTS = 12 SUBSET_SIZE = 300 def category_quotas(pool: list[dict], n: int) -> dict[str, int]: groups = defaultdict(list) for i, row in enumerate(pool): groups[row["Category"]].append(i) total = len(pool) alloc = {g: int(round(n * len(ids) / total)) for g, ids in groups.items()} diff = n - sum(alloc.values()) order = sorted(groups.keys(), key=lambda g: len(groups[g]), reverse=True) j = 0 while diff != 0 and order: g = order[j % len(order)] if diff > 0: alloc[g] += 1 diff -= 1 else: if alloc[g] > 0: alloc[g] -= 1 diff += 1 j += 1 return alloc def indices_by_category(pool: list[dict]) -> dict[str, list[int]]: out: dict[str, list[int]] = defaultdict(list) for i, row in enumerate(pool): out[row["Category"]].append(i) return out def individual_from_quotas(by_cat: dict[str, list[int]], quotas: dict[str, int], rng: random.Random) -> list[int]: chosen: list[int] = [] for cat, k in quotas.items(): ids = by_cat[cat] k = min(k, len(ids)) chosen.extend(rng.sample(ids, k)) all_idx = [i for ids in by_cat.values() for i in ids] if len(chosen) < SUBSET_SIZE: remain = list(set(all_idx) - set(chosen)) chosen.extend(rng.sample(remain, SUBSET_SIZE - len(chosen))) elif len(chosen) > SUBSET_SIZE: chosen = rng.sample(chosen, SUBSET_SIZE) return sorted(chosen) def mutate(ind: list[int], by_cat: dict[str, list[int]], quotas: dict[str, int], rng: random.Random) -> list[int]: """Swap members within categories at mutation_rate while preserving quotas.""" ind_set = set(ind) # group current members by category — need pool categories; use reverse map cat_of = {} for cat, ids in by_cat.items(): for i in ids: cat_of[i] = cat new = list(ind) for pos, idx in enumerate(list(new)): if rng.random() > MUTATION_RATE: continue cat = cat_of[idx] candidates = [j for j in by_cat[cat] if j not in ind_set] if not candidates: continue repl = rng.choice(candidates) ind_set.discard(idx) ind_set.add(repl) new[pos] = repl return sorted(new) def crossover(a: list[int], b: list[int], by_cat: dict[str, list[int]], quotas: dict[str, int], rng: random.Random) -> list[int]: """Category-wise uniform crossover then repair quotas.""" cat_of = {} for cat, ids in by_cat.items(): for i in ids: cat_of[i] = cat child: list[int] = [] for cat, k in quotas.items(): a_cat = [i for i in a if cat_of[i] == cat] b_cat = [i for i in b if cat_of[i] == cat] pool_ids = list(dict.fromkeys(a_cat + b_cat)) if len(pool_ids) < k: extras = [j for j in by_cat[cat] if j not in pool_ids] pool_ids.extend(extras) child.extend(rng.sample(pool_ids, min(k, len(pool_ids)))) if len(child) < SUBSET_SIZE: all_idx = [i for ids in by_cat.values() for i in ids] remain = list(set(all_idx) - set(child)) child.extend(rng.sample(remain, SUBSET_SIZE - len(child))) elif len(child) > SUBSET_SIZE: child = rng.sample(child, SUBSET_SIZE) return sorted(child) def fitness(indices: list[int], pool, full_metrics, dfa_cols, sigma) -> float: subset = [pool[i] for i in indices] sub_m = metrics(subset, dfa_cols, sigma) return float(compare(full_metrics, sub_m, dfa_cols)["F_score"]) def evolve_one_start(pool, full_metrics, dfa_cols, sigma, seed: int) -> tuple[list[int], float, dict]: rng = random.Random(seed) quotas = category_quotas(pool, SUBSET_SIZE) by_cat = indices_by_category(pool) population = [individual_from_quotas(by_cat, quotas, rng) for _ in range(POP_SIZE)] strat = sample_stratified(pool, SUBSET_SIZE, "Category", rng) id_map = {id(r): i for i, r in enumerate(pool)} population[0] = sorted(id_map[id(r)] for r in strat) scores = [fitness(ind, pool, full_metrics, dfa_cols, sigma) for ind in population] best_idx = int(np.argmin(scores)) best_ind = list(population[best_idx]) best_score = scores[best_idx] stagnant = 0 history = [] for gen in range(N_GENS): order = sorted(range(POP_SIZE), key=lambda i: scores[i]) elites = [list(population[i]) for i in order[:ELITISM]] new_pop = elites[:] while len(new_pop) < POP_SIZE: # tournament cands = rng.sample(range(POP_SIZE), 3) p1 = population[min(cands, key=lambda i: scores[i])] cands = rng.sample(range(POP_SIZE), 3) p2 = population[min(cands, key=lambda i: scores[i])] child = crossover(p1, p2, by_cat, quotas, rng) child = mutate(child, by_cat, quotas, rng) new_pop.append(child) population = new_pop scores = [fitness(ind, pool, full_metrics, dfa_cols, sigma) for ind in population] gen_best_i = int(np.argmin(scores)) gen_best = scores[gen_best_i] history.append({"gen": gen, "best_F": gen_best}) if gen_best + 1e-12 < best_score: best_score = gen_best best_ind = list(population[gen_best_i]) stagnant = 0 else: stagnant += 1 if stagnant >= EARLY_STOP: break cmp = compare(full_metrics, metrics([pool[i] for i in best_ind], dfa_cols, sigma), dfa_cols) return best_ind, best_score, {"history": history, "compare": cmp, "seed": seed} def main(): ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--out-dir", type=str, default=str(CODE_ROOT / "configs" / "evolutionary_runs")) ap.add_argument("--n-starts", type=int, default=N_STARTS) ap.add_argument("--seed0", type=int, default=0) args = ap.parse_args() out = Path(args.out_dir) out.mkdir(parents=True, exist_ok=True) pool, _optimized, sigma, dfa_cols = load_data() full_metrics = metrics(pool, dfa_cols, sigma) results = [] global_best = None global_score = float("inf") for s in range(args.n_starts): seed = args.seed0 + s print(f"[start {s+1}/{args.n_starts}] seed={seed}", flush=True) ind, score, meta = evolve_one_start(pool, full_metrics, dfa_cols, sigma, seed) results.append({"seed": seed, "F_score": score, **meta["compare"]}) if score < global_score: global_score = score global_best = ind with open(out / "best_subset_reactions.json", "w") as f: json.dump( { "seed": seed, "F_score": score, "compare": meta["compare"], "reactions": [pool[i]["Reaction"] for i in ind], }, f, indent=2, ) with open(out / "multi_start_summary.csv", "w", newline="") as f: w = csv.DictWriter(f, fieldnames=["seed", "F_score", "Delta_MAD", "Delta_NER", "Delta_max"]) w.writeheader() for r in results: w.writerow( { "seed": r["seed"], "F_score": r["F_score"], "Delta_MAD": r["Delta_MAD"], "Delta_NER": r["Delta_NER"], "Delta_max": r["Delta_max"], } ) print(f"Best F={global_score:.6f}; wrote {out}", flush=True) if __name__ == "__main__": main()