feat: add comparative FIRT-ORPO benchmark runner for AdamW vs FCES
This commit is contained in:
AI-anonymous
108
benchmark_results.json
Normal file
108
benchmark_results.json
Normal file
@@ -0,0 +1,108 @@
|
||||
{
|
||||
"model": "EleutherAI/pythia-70m",
|
||||
"steps": 5,
|
||||
"adam_metrics": [
|
||||
{
|
||||
"step": 0,
|
||||
"loss": 7.041037559509277,
|
||||
"sft_loss": 6.995092391967773,
|
||||
"wall_clock_time": 0.8969566999949166,
|
||||
"tokens_processed": 35
|
||||
},
|
||||
{
|
||||
"step": 1,
|
||||
"loss": 2.5148799419403076,
|
||||
"sft_loss": 2.4989213943481445,
|
||||
"wall_clock_time": 1.8027480999953696,
|
||||
"tokens_processed": 70
|
||||
},
|
||||
{
|
||||
"step": 2,
|
||||
"loss": 0.8542575836181641,
|
||||
"sft_loss": 0.8379661440849304,
|
||||
"wall_clock_time": 2.7821450999981607,
|
||||
"tokens_processed": 105
|
||||
},
|
||||
{
|
||||
"step": 3,
|
||||
"loss": 0.6839541792869568,
|
||||
"sft_loss": 0.667436420917511,
|
||||
"wall_clock_time": 3.560283499995421,
|
||||
"tokens_processed": 140
|
||||
},
|
||||
{
|
||||
"step": 4,
|
||||
"loss": 0.359605997800827,
|
||||
"sft_loss": 0.3395426273345947,
|
||||
"wall_clock_time": 4.330241899995599,
|
||||
"tokens_processed": 175
|
||||
}
|
||||
],
|
||||
"fces_metrics": [
|
||||
{
|
||||
"step": 0,
|
||||
"loss": 7.041037559509277,
|
||||
"sft_loss": 6.995092391967773,
|
||||
"wall_clock_time": 13.06952640000236,
|
||||
"tokens_processed": 35
|
||||
},
|
||||
{
|
||||
"step": 1,
|
||||
"loss": 6.717805862426758,
|
||||
"sft_loss": 6.71108341217041,
|
||||
"wall_clock_time": 20.157852299998922,
|
||||
"tokens_processed": 70
|
||||
},
|
||||
{
|
||||
"step": 2,
|
||||
"loss": 14.43155574798584,
|
||||
"sft_loss": 14.42345142364502,
|
||||
"wall_clock_time": 27.273785699995642,
|
||||
"tokens_processed": 105
|
||||
},
|
||||
{
|
||||
"step": 3,
|
||||
"loss": 7.063178062438965,
|
||||
"sft_loss": 7.0147223472595215,
|
||||
"wall_clock_time": 34.50635989999864,
|
||||
"tokens_processed": 140
|
||||
},
|
||||
{
|
||||
"step": 4,
|
||||
"loss": 5.816511631011963,
|
||||
"sft_loss": 5.8062052726745605,
|
||||
"wall_clock_time": 42.63929800000187,
|
||||
"tokens_processed": 175
|
||||
}
|
||||
],
|
||||
"pre_eval": [
|
||||
{
|
||||
"case_id": 1,
|
||||
"output": "Case: Mieter M besichtigt im Januar eine Wohnung des Vermieters V. Schimmelfleck im Schlafzimmer. V sagt er wird es 'irgendwann mal wegmachen'. M unterschreibt Mietvertrag vorbehaltlos und zieht am 1. Februar ein. Im M\u00e4rz mindert M die Miete selbstst\u00e4ndig um 20% wegen des Schimmels. V verlangt volle Miete.\nStatutes: \u00a7 535 Abs. 2 BGB (Mietzahlung), \u00a7 536 Abs. 1 BGB (Mietminderung bei Sachmangel), \u00a7 536b BGB (Kenntnis des Mieters bei Vertragsschluss).\nAnalyze and resolve using Flaw Induced Reasoning Traces (FIRT):\nT0 Draft:\nThe following is a summary of the relevant facts and the relevant facts.\n1. The following is a summary of the relevant facts and the relevant facts.\n2. The following is a summary of the relevant facts and the relevant facts.\n3. The following is a summary of the relevant facts and the relevant facts.\n4. The following is a summary of the relevant facts and the relevant facts.\n5. The following is a summary of the relevant facts and the relevant facts.\n6. The following is a summary of the relevant facts and the relevant facts.\n7. The following is a summary of the relevant facts and the relevant facts.\n8. The following is a summary of the relevant facts and the relevant facts.\n9. The following is a summary of the relevant facts and the relevant facts.\n10. The following is a summary of the relevant facts and the relevant facts.\n11. The following is a summary of the relevant facts and the relevant facts.\n12. The following is a summary of the relevant facts and the relevant facts.\n13. The following is a summary of the relevant facts and the relevant facts.\n14. The following is a summary of the relevant facts and the relevant facts.\n15. The"
|
||||
},
|
||||
{
|
||||
"case_id": 2,
|
||||
"output": "Case: Toilettensp\u00fclung bei F f\u00e4llt am Freitagabend aus, Wasser l\u00e4uft ununterbrochen und droht \u00fcberzulaufen. V ist auf Segeltrip und nicht erreichbar. Kein Hausmeister. F beauftragt am Samstagmorgen den Notdienst K, bezahlt 300 \u20ac und fordert Erstattung von V. V weigert sich.\nStatutes: \u00a7 536a Abs. 2 Nr. 2 BGB (Selbstbeseitigungsrecht bei Gefahr im Verzug), \u00a7 535 BGB.\nAnalyze and resolve using Flaw Induced Reasoning Traces (FIRT):\nT0 Draft:\n\"The FIRT is a method of analysis of the nature of the FIRT. It is a method of analysis of the nature of the FIRT. It is a method of analysis of the nature of the FIRT. It is a method of analysis of the nature of the FIRT. It is a method of analysis of the nature of the FIRT. It is a method of analysis of the nature of the FIRT. It is a method of analysis of the nature of the FIRT. It is a method of analysis of the nature of the FIRT. It is a method of analysis of the nature of the FIRT. It is a method of analysis of the nature of the FIRT. It is a method of analysis of the nature of the FIRT. It is a method of analysis of the nature of the FIRT. It is a method of analysis of the nature of the FIRT. It is a method of analysis of the nature of the FIRT. It is a method of analysis of the nature of the FIRT. It is a method of analysis of the nature of the FIRT. It is a method of analysis of the nature of the FIRT. It is a method of analysis of the nature of the FIRT. It"
|
||||
}
|
||||
],
|
||||
"adam_post_eval": [
|
||||
{
|
||||
"case_id": 1,
|
||||
"output": "Case: Mieter M besichtigt im Januar eine Wohnung des Vermieters V. Schimmelfleck im Schlafzimmer. V sagt er wird es 'irgendwann mal wegmachen'. M unterschreibt Mietvertrag vorbehaltlos und zieht am 1. Februar ein. Im M\u00e4rz mindert M die Miete selbstst\u00e4ndig um 20% wegen des Schimmels. V verlangt volle Miete.\nStatutes: \u00a7 535 Abs. 2 BGB (Mietzahlung), \u00a7 536 Abs. 1 BGB (Mietminderung bei Sachmangel), \u00a7 536b BGB (Kenntnis des Mieters bei Vertragsschluss).\nAnalyze and resolve using Flaw Induced Reasoning Traces (FIRT):\nT0 Draft:\n \u00a7 536a BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB. \u00a7 536 BGB."
|
||||
},
|
||||
{
|
||||
"case_id": 2,
|
||||
"output": "Case: Toilettensp\u00fclung bei F f\u00e4llt am Freitagabend aus, Wasser l\u00e4uft ununterbrochen und droht \u00fcberzulaufen. V ist auf Segeltrip und nicht erreichbar. Kein Hausmeister. F beauftragt am Samstagmorgen den Notdienst K, bezahlt 300 \u20ac und fordert Erstattung von V. V weigert sich.\nStatutes: \u00a7 536a Abs. 2 Nr. 2 BGB (Selbstbeseitigungsrecht bei Gefahr im Verzug), \u00a7 535 BGB.\nAnalyze and resolve using Flaw Induced Reasoning Traces (FIRT):\nT0 Draft:\n \u00a7 536a BGB. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined opinion correct. Refined correct. Refined opinion correct. Refined correct. Refined opinion correct. Refined correct. Refined correct. Refined correct. Refined correct. Refined correct. Refined correct. Refined correct. Refined correct. Refined correct. Refined correct. Refined correct. Refined correct. Refined correct. Refined correct. Refined correct. Refined correct. Refined correct. Refined correct. Refined correct. Refined correct. Refined correct. Refined correct. Refined correct."
|
||||
}
|
||||
],
|
||||
"fces_post_eval": [
|
||||
{
|
||||
"case_id": 1,
|
||||
"output": "Case: Mieter M besichtigt im Januar eine Wohnung des Vermieters V. Schimmelfleck im Schlafzimmer. V sagt er wird es 'irgendwann mal wegmachen'. M unterschreibt Mietvertrag vorbehaltlos und zieht am 1. Februar ein. Im M\u00e4rz mindert M die Miete selbstst\u00e4ndig um 20% wegen des Schimmels. V verlangt volle Miete.\nStatutes: \u00a7 535 Abs. 2 BGB (Mietzahlung), \u00a7 536 Abs. 1 BGB (Mietminderung bei Sachmangel), \u00a7 536b BGB (Kenntnis des Mieters bei Vertragsschluss).\nAnalyze and resolve using Flaw Induced Reasoning Traces (FIRT):\nT0 Draft:\n................................................................................................................................................................................. \u00a7. \u00a7. \u00a7........................................................ \u00a7. \u00a7. \u00a7............."
|
||||
},
|
||||
{
|
||||
"case_id": 2,
|
||||
"output": "Case: Toilettensp\u00fclung bei F f\u00e4llt am Freitagabend aus, Wasser l\u00e4uft ununterbrochen und droht \u00fcberzulaufen. V ist auf Segeltrip und nicht erreichbar. Kein Hausmeister. F beauftragt am Samstagmorgen den Notdienst K, bezahlt 300 \u20ac und fordert Erstattung von V. V weigert sich.\nStatutes: \u00a7 536a Abs. 2 Nr. 2 BGB (Selbstbeseitigungsrecht bei Gefahr im Verzug), \u00a7 535 BGB.\nAnalyze and resolve using Flaw Induced Reasoning Traces (FIRT):\nT0 Draft:\n.......................................................................................................................................... \u00a7.................... \u00a7.......... \u00a7......... \u00a7....... \u00a7........ \u00a7........ \u00a7......... \u00a7.......... \u00a7......... \u00a7......... \u00a7........"
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user