diff --git a/benchmark_fces_vs_adam.py b/benchmark_fces_vs_adam.py index 6760593..17facfa 100644 --- a/benchmark_fces_vs_adam.py +++ b/benchmark_fces_vs_adam.py @@ -16,6 +16,11 @@ from send_telemetry import push_to_mariadb, push_to_surrealdb # noqa: E402 from transformers import AutoModelForCausalLM, AutoTokenizer # noqa: E402 from parasitic_qlora import ParasiticQLoRAExtractor, QLoRAConfig # noqa: E402 from expert_manifold_alignment import ExpertManifoldAligner # noqa: E402 +from representation_engineering import ( # noqa: E402 + PlaybookParser, + RepresentationVectorExtractor, + SkillVectorLibrary, +) # ============================================================================== # 1. DSPY SIGNATURE & SYSTEM DESIGN @@ -278,6 +283,27 @@ def train_run( library_path = f"parasitic_adapters_{optimizer_name.lower()}_step{steps}.pt" extractor.save_library(library_path) + # Extract and save Skill representation vectors + print(f"[{optimizer_name}] Extracting Skill representation vectors...") + try: + skills_dir = "C:/Users/Sven/Documents/svenco-knowledge/skills" + playbooks = PlaybookParser.parse_directory(skills_dir) + print(f"[{optimizer_name}] Found {len(playbooks)} playbooks in {skills_dir}") + + rep_extractor = RepresentationVectorExtractor(model, tokenizer, device) + skill_lib = SkillVectorLibrary() + + for pb in playbooks: + print(f"[{optimizer_name}] Extracting steering vector for skill: {pb.name}") + vec = rep_extractor.extract_steering_vector(pb) + skill_lib.add_vector(vec) + + skill_lib_path = f"skill_library_{optimizer_name.lower()}.pt" + skill_lib.save(skill_lib_path) + print(f"[{optimizer_name}] Saved SkillVectorLibrary to {skill_lib_path}") + except Exception as e: + print(f"[{optimizer_name}] Error extracting skill representation vectors: {e}") + # 4. Post-Training Evaluation print(f"[{optimizer_name}] Running Post-Training Evaluation...") post_eval = evaluate_model(model, tokenizer, device)