feat: more blunder tests

This commit is contained in:
Krzysztof kuhy Rudnicki 2025-08-23 15:53:42 +02:00
parent 7f02645ca6
commit 78b6c3c053
6 changed files with 280 additions and 21 deletions

View File

@ -1 +1 @@
17
18

View File

@ -0,0 +1,49 @@
import os
import sys
import chess
import pytest
# Ensure repo root is importable when running pytest directly
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
if REPO_ROOT not in sys.path:
sys.path.insert(0, REPO_ROOT)
from PYTHON.lichess_bot.engine import RandomEngine # noqa: E402
BLUNDER_CASES = [
("rnb1kb1r/pp3ppp/4q3/2p3N1/4p3/8/PPPPNPPP/R1BQ1RK1 b kq - 1 9", "e6a2", "ply18_B_e6a2"),
("2kr3r/1p4p1/4bp2/7p/Q2b1B2/2P5/1P3PPP/5RK1 w - - 0 20", "c3d4", "ply39_W_c3d4"),
("2kr3r/1p4p1/4bp2/7p/Q2P1B2/8/1P3PPP/5RK1 b - - 0 20", "e6d5", "ply40_B_e6d5"),
("2kr3r/1p4p1/5p2/3b3p/Q2P1B2/8/1P3PPP/5RK1 w - - 1 21", "a4a8", "ply41_W_a4a8"),
("3r3r/1p1k2p1/5p2/3b3p/Q2P1B2/8/1P3PPP/5RK1 b - - 4 22", "d7c8", "ply44_B_d7c8"),
("2kr3r/1p4p1/5p2/Q2b3p/3P1B2/8/1P3PPP/5RK1 b - - 6 23", "b7b6", "ply46_B_b7b6"),
("2kr3r/6p1/1Q3p2/3b3p/3P1B2/8/1P3PPP/5RK1 b - - 0 24", "c8d7", "ply48_B_c8d7"),
("3r3r/3k2p1/1Q3p2/3b3p/3P1B2/8/1P3PPP/5RK1 w - - 1 25", "f4c7", "ply49_W_f4c7"),
("3r3r/2Bk2p1/1Q3p2/3b3p/3P4/8/1P3PPP/5RK1 b - - 2 25", "d8c8", "ply50_B_d8c8"),
("2r4r/2Bk2p1/1Q3p2/3b3p/3P4/8/1P3PPP/5RK1 w - - 3 26", "c7b8", "ply51_W_c7b8"),
("1r5r/Q5p1/4kp2/3b3p/3P4/8/1P3PPP/4R1K1 b - - 3 28", "e6d6", "ply56_B_e6d6"),
("1r5r/2k1R1p1/5p2/3Q3p/3P4/8/1P3PPP/6K1 b - - 2 31", "c7c8", "ply62_B_c7c8"),
("1rk4r/4R1p1/5p2/3Q3p/3P4/8/1P3PPP/6K1 w - - 3 32", "d5c5", "ply63_W_d5c5"),
("1r1k3r/4R1p1/5p2/2Q4p/3P4/8/1P3PPP/6K1 w - - 5 33", "d4d5", "ply65_W_d4d5"),
("3k3r/1Q2R3/5pp1/3P3p/8/8/1P3PPP/6K1 w - - 0 37", "d5d6", "ply73_W_d5d6"),
("3k3r/1Q2R3/3P1p2/6pp/8/8/1P3PPP/6K1 w - - 0 38", "b7b8", "ply75_W_b7b8"),
]
@pytest.mark.parametrize('fen,blunder_uci,label', BLUNDER_CASES, ids=[c[2] for c in BLUNDER_CASES])
def test_engine_avoids_logged_blunder(fen, blunder_uci, label):
board = chess.Board(fen)
eng = RandomEngine(depth=4, max_time_sec=1.2)
# Prefer explanation variant if available for better failure messages
move = None
explanation = ''
if hasattr(eng, 'choose_move_with_explanation'):
try:
mv, expl = eng.choose_move_with_explanation(board, time_budget_sec=1.2)
move, explanation = mv, expl or ''
except Exception:
move = eng.choose_move(board)
else:
move = eng.choose_move(board)
assert move is not None, 'Engine returned no move'
assert move in board.legal_moves, 'Engine move is illegal'
assert move.uci() != blunder_uci, f'Engine repeated blunder {blunder_uci} at {label}. Explanation: {explanation}'

View File

@ -0,0 +1,62 @@
import os
import sys
import chess
import pytest
# Ensure repo root is importable when running pytest directly
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
if REPO_ROOT not in sys.path:
sys.path.insert(0, REPO_ROOT)
from PYTHON.lichess_bot.engine import RandomEngine # noqa: E402
BLUNDER_CASES = [
("rnb2rk1/pp2bppp/8/8/2qN4/2N5/PPP2PPP/R1BQK2R w - - 3 13", "b2b3", "ply25_W_b2b3"),
("rn3rk1/pp2bppp/8/3Q1b2/8/1P6/2q2PPP/2B2K1R w - - 0 18", "d5b7", "ply35_W_d5b7"),
("rn3rk1/pQ2bppp/8/5b2/8/1P6/2q2PPP/2B2K1R b - - 0 18", "c2f2", "ply36_B_c2f2"),
("6k1/p2n1ppp/8/2b5/6b1/1P6/3K3P/4R3 b - - 0 27", "c5b4", "ply54_B_c5b4"),
("6k1/p2n1ppp/8/8/5Kb1/1P6/7P/4b3 b - - 1 29", "e1c3", "ply58_B_e1c3"),
("6k1/p2n1ppp/8/8/6K1/1Pb5/7P/8 b - - 0 30", "d7f6", "ply60_B_d7f6"),
("6k1/p4ppp/5n2/5K2/8/1Pb5/7P/8 b - - 2 31", "f6d7", "ply62_B_f6d7"),
("6k1/p2n1ppp/8/5K2/8/1Pb5/7P/8 w - - 3 32", "f5e4", "ply63_W_f5e4"),
("6k1/p2n1ppp/8/8/4K3/1Pb5/7P/8 b - - 4 32", "d7f6", "ply64_B_d7f6"),
("6k1/p4ppp/8/4b2n/4K3/1P5P/8/8 b - - 2 35", "e5b8", "ply70_B_e5b8"),
("1b4k1/p4ppp/8/7n/4K3/1P5P/8/8 w - - 3 36", "e4f5", "ply71_W_e4f5"),
("1b4k1/p4ppp/8/5K1n/8/1P5P/8/8 b - - 4 36", "b8d6", "ply72_B_b8d6"),
("6k1/p4ppp/3b4/5K1n/8/1P5P/8/8 w - - 5 37", "f5g5", "ply73_W_f5g5"),
("6k1/p4ppp/3b4/6Kn/8/1P5P/8/8 b - - 6 37", "h5f4", "ply74_B_h5f4"),
("6k1/p4ppp/3b4/8/5nK1/1P5P/8/8 b - - 8 38", "h7h5", "ply76_B_h7h5"),
("6k1/p4pp1/3b4/7p/5nK1/1P5P/8/8 w - - 0 39", "g4g3", "ply77_W_g4g3"),
("6k1/p4pp1/3b4/7p/5n2/1P4KP/8/8 b - - 1 39", "f4e6", "ply78_B_f4e6"),
("6k1/p4pp1/3bn3/7p/8/1P4KP/8/8 w - - 2 40", "g3h4", "ply79_W_g3h4"),
("6k1/p4pp1/3bn3/7p/7K/1P5P/8/8 b - - 3 40", "e6c5", "ply80_B_e6c5"),
("6k1/p4pp1/3b4/2n4p/7K/1P5P/8/8 w - - 4 41", "h4h5", "ply81_W_h4h5"),
("6k1/p4pp1/3b4/2n4K/8/1P5P/8/8 b - - 0 41", "c5b3", "ply82_B_c5b3"),
("6k1/p4pp1/3b4/6K1/8/1n5P/8/8 b - - 1 42", "d6e7", "ply84_B_d6e7"),
("6k1/p3bpp1/8/5K2/8/1n5P/8/8 b - - 3 43", "b3d4", "ply86_B_b3d4"),
("8/p3kpp1/5b2/8/3nK3/7P/8/8 w - - 10 47", "e4f4", "ply93_W_e4f4"),
("8/p3kpp1/4nb2/8/5K2/7P/8/8 w - - 12 48", "f4f5", "ply95_W_f4f5"),
("6k1/p4pp1/5b2/3K4/3n4/7P/8/8 w - - 18 51", "d5d6", "ply101_W_d5d6"),
("6k1/p4pp1/4nb2/2K5/8/7P/8/8 w - - 24 54", "c5d6", "ply107_W_c5d6"),
("6k1/p4pp1/3K1b2/8/5n2/7P/8/8 w - - 26 55", "d6d7", "ply109_W_d6d7"),
("6k1/p2K1pp1/5b2/8/8/7n/8/8 w - - 0 56", "d7e8", "ply111_W_d7e8"),
]
@pytest.mark.parametrize('fen,blunder_uci,label', BLUNDER_CASES, ids=[c[2] for c in BLUNDER_CASES])
def test_engine_avoids_logged_blunder(fen, blunder_uci, label):
board = chess.Board(fen)
eng = RandomEngine(depth=4, max_time_sec=1.2)
# Prefer explanation variant if available for better failure messages
move = None
explanation = ''
if hasattr(eng, 'choose_move_with_explanation'):
try:
mv, expl = eng.choose_move_with_explanation(board, time_budget_sec=1.2)
move, explanation = mv, expl or ''
except Exception:
move = eng.choose_move(board)
else:
move = eng.choose_move(board)
assert move is not None, 'Engine returned no move'
assert move in board.legal_moves, 'Engine move is illegal'
assert move.uci() != blunder_uci, f'Engine repeated blunder {blunder_uci} at {label}. Explanation: {explanation}'

View File

@ -0,0 +1,42 @@
import os
import sys
import chess
import pytest
# Ensure repo root is importable when running pytest directly
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
if REPO_ROOT not in sys.path:
sys.path.insert(0, REPO_ROOT)
from PYTHON.lichess_bot.engine import RandomEngine # noqa: E402
BLUNDER_CASES = [
("r1bqk2r/ppp2ppp/2np1n2/2b1p1B1/2B1P3/3P1N2/PPP2PPP/RN1Q1RK1 b kq - 1 6", "c5f2", "ply12_B_c5f2"),
("r1bq1rk1/pp3ppp/3p1n2/2p1p1B1/2P1P3/2PP1N2/P5PP/RN1Q1RK1 b - - 0 11", "f6e4", "ply22_B_f6e4"),
("3r1r2/pp3ppk/3N3p/2p1N3/4P3/2P5/P5PP/R2Q1RK1 b - - 0 17", "h7g8", "ply34_B_h7g8"),
("3r1rk1/pp3pp1/3N3p/2p1N3/4P3/2P5/P5PP/R2Q1RK1 w - - 1 18", "e5f7", "ply35_W_e5f7"),
("3r1rk1/pp3Np1/3N3p/2p5/4P3/2P5/P5PP/R2Q1RK1 b - - 0 18", "f8f7", "ply36_B_f8f7"),
("5Q2/p5pk/4P2p/1pp5/8/2P5/P5PP/5RK1 b - - 0 24", "h7g6", "ply48_B_h7g6"),
("5Q2/p5p1/4P1kp/1pp5/8/2P5/P5PP/5RK1 w - - 1 25", "e6e7", "ply49_W_e6e7"),
("5Q2/p3P1p1/6kp/1pp5/8/2P5/P5PP/5RK1 b - - 0 25", "g6h7", "ply50_B_g6h7"),
("4QQ2/p6k/6pp/1pp5/8/2P5/P5PP/5RK1 w - - 0 27", "f8h8", "ply53_W_f8h8"),
]
@pytest.mark.parametrize('fen,blunder_uci,label', BLUNDER_CASES, ids=[c[2] for c in BLUNDER_CASES])
def test_engine_avoids_logged_blunder(fen, blunder_uci, label):
board = chess.Board(fen)
eng = RandomEngine(depth=4, max_time_sec=1.2)
# Prefer explanation variant if available for better failure messages
move = None
explanation = ''
if hasattr(eng, 'choose_move_with_explanation'):
try:
mv, expl = eng.choose_move_with_explanation(board, time_budget_sec=1.2)
move, explanation = mv, expl or ''
except Exception:
move = eng.choose_move(board)
else:
move = eng.choose_move(board)
assert move is not None, 'Engine returned no move'
assert move in board.legal_moves, 'Engine move is illegal'
assert move.uci() != blunder_uci, f'Engine repeated blunder {blunder_uci} at {label}. Explanation: {explanation}'

View File

@ -0,0 +1,43 @@
import os
import sys
import chess
import pytest
# Ensure repo root is importable when running pytest directly
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
if REPO_ROOT not in sys.path:
sys.path.insert(0, REPO_ROOT)
from PYTHON.lichess_bot.engine import RandomEngine # noqa: E402
BLUNDER_CASES = [
("r4r2/p2p3k/n2Np1p1/q1p5/P5Q1/8/3NKP2/8 b - - 2 24", "a5c7", "ply48_B_a5c7"),
("r4N2/p6k/n5pq/P1pp4/6Q1/5N2/4KP2/8 b - - 0 30", "h6f8", "ply60_B_h6f8"),
("r4q2/p6k/n5p1/P1pp4/6Q1/5N2/4KP2/8 w - - 0 31", "g4h3", "ply61_W_g4h3"),
("r4qk1/p2Q4/n5p1/P1pp4/8/5N2/4KP2/8 w - - 4 33", "e2e3", "ply65_W_e2e3"),
("4rqk1/p2Q4/n5p1/P1pp4/8/4KN2/5P2/8 w - - 6 34", "e3d2", "ply67_W_e3d2"),
("4rqk1/p2Q4/n5p1/P1pp4/8/5N2/3K1P2/8 b - - 7 34", "d5d4", "ply68_B_d5d4"),
("4rqk1/p2Q4/n5p1/P1p5/3p4/5N2/3K1P2/8 w - - 0 35", "d7a7", "ply69_W_d7a7"),
("4r1k1/Q7/n5p1/P1p5/3p4/5q2/3K1P2/8 w - - 0 36", "d2c2", "ply71_W_d2c2"),
("6k1/Q7/n5p1/P1p5/3p4/8/4rq2/3K4 w - - 0 38", "d1c1", "ply75_W_d1c1"),
("6k1/Q7/n5p1/P1p5/3p4/8/4rq2/2K5 b - - 1 38", "f2e1", "ply76_B_f2e1"),
]
@pytest.mark.parametrize('fen,blunder_uci,label', BLUNDER_CASES, ids=[c[2] for c in BLUNDER_CASES])
def test_engine_avoids_logged_blunder(fen, blunder_uci, label):
board = chess.Board(fen)
eng = RandomEngine(depth=4, max_time_sec=1.2)
# Prefer explanation variant if available for better failure messages
move = None
explanation = ''
if hasattr(eng, 'choose_move_with_explanation'):
try:
mv, expl = eng.choose_move_with_explanation(board, time_budget_sec=1.2)
move, explanation = mv, expl or ''
except Exception:
move = eng.choose_move(board)
else:
move = eng.choose_move(board)
assert move is not None, 'Engine returned no move'
assert move in board.legal_moves, 'Engine move is illegal'
assert move.uci() != blunder_uci, f'Engine repeated blunder {blunder_uci} at {label}. Explanation: {explanation}'

View File

@ -1,18 +1,32 @@
#!/usr/bin/env python3
"""
Generate pytest cases from a lichess analysis log.
Generate pytest cases from one or more lichess analysis logs.
Input: a log file that contains a "Columns:" section and a "PGN:" section,
like the example the user provided. We'll extract each row where class==Blunder,
reconstruct the FEN of the position before the blunder, and the blunder move in
UCI. Then we write a parametrized pytest that asserts the engine does not pick
that same blunder move from that position.
Input: log files that contain a "Columns:" section and a "PGN:" section.
We'll extract each row where class==Blunder, reconstruct the FEN of the
position before the blunder, and the blunder move in UCI. Then we'll write
parametrized pytest files that assert the engine does not pick that same
blunder move from those positions.
Usage:
python PYTHON/lichess_bot/tools/generate_blunder_tests.py /path/to/lichess_bot_game_xxxxx.log
Where logs are loaded from:
- By default (no arguments), all logs in the "past_games" folder located
next to this script will be processed (files matching lichess_bot_game_*.log).
- If a single argument is provided and it's a file path, that file is used.
- If a single argument looks like a game id (e.g. OVmR29MI), the script will
look for past_games/lichess_bot_game_<gameid>.log next to this script.
It will create a file like:
PYTHON/lichess_bot/tests/test_blunders_<gameid>.py
Usage examples:
# Process all logs in tools/past_games
python PYTHON/lichess_bot/tools/generate_blunder_tests.py
# Process a specific game by id from tools/past_games
python PYTHON/lichess_bot/tools/generate_blunder_tests.py OVmR29MI
# Process an explicit file path
python PYTHON/lichess_bot/tools/generate_blunder_tests.py /path/to/lichess_bot_game_xxxxx.log
It will create files like:
PYTHON/lichess_bot/tests/test_blunders_<gameid>.py
Dependencies: python-chess, pytest (already in requirements.txt)
"""
@ -168,30 +182,30 @@ BLUNDER_CASES = [
print(f"Wrote {target_path} with {len(cases)} blunder checks (game {game_id}).")
def main(argv: List[str]) -> int:
if len(argv) < 2:
print("Usage: generate_blunder_tests.py /path/to/lichess_bot_game_xxx.log")
def _process_single_log(log_path: str) -> int:
"""Process a single log file. Returns 0 on success, non-zero otherwise."""
try:
with open(log_path, "r", encoding="utf-8") as fh:
text = fh.read()
except FileNotFoundError:
print(f"Log file not found: {log_path}")
return 2
log_path = argv[1]
with open(log_path, "r", encoding="utf-8") as fh:
text = fh.read()
blunders = parse_columns_for_blunders(text)
if not blunders:
print("No blunders found in the log's Columns section.")
print(f"No blunders found in Columns section: {os.path.basename(log_path)}")
return 1
pgn_text = extract_pgn(text)
if not pgn_text:
print("No PGN section found in the log.")
print(f"No PGN section found: {os.path.basename(log_path)}")
return 1
cases = fen_and_uci_for_blunders(pgn_text, blunders)
if not cases:
print("Failed to reconstruct any blunder positions from PGN.")
print(f"Failed to reconstruct any blunder positions from PGN: {os.path.basename(log_path)}")
return 1
# Try to derive game id from file name
base = os.path.basename(log_path)
m = re.search(r"game_([A-Za-z0-9]+)\.log$", base)
game_id = m.group(1) if m else os.path.splitext(base)[0]
@ -202,5 +216,54 @@ def main(argv: List[str]) -> int:
return 0
def main(argv: List[str]) -> int:
script_dir = os.path.dirname(__file__)
past_dir = os.path.abspath(os.path.join(script_dir, "past_games"))
# No argument: process all logs in past_games
if len(argv) == 1:
if not os.path.isdir(past_dir):
print(f"No past_games directory found at {past_dir}")
return 2
logs = [
os.path.join(past_dir, name)
for name in os.listdir(past_dir)
if re.match(r"lichess_bot_game_[A-Za-z0-9]+\.log$", name)
]
if not logs:
print(f"No logs found in {past_dir}")
return 1
# Sort by mtime ascending for determinism
logs.sort(key=lambda p: os.path.getmtime(p))
ok = 0
for lp in logs:
rc = _process_single_log(lp)
if rc == 0:
ok += 1
print(f"Processed {len(logs)} logs from {past_dir}, succeeded: {ok}, failed: {len(logs)-ok}")
return 0 if ok > 0 else 1
# One argument: game id or file path
arg = argv[1]
candidate_path = None
if os.path.isfile(arg):
candidate_path = arg
else:
# Treat as game id, resolve within past_games
if re.fullmatch(r"[A-Za-z0-9]+", arg):
candidate_path = os.path.join(past_dir, f"lichess_bot_game_{arg}.log")
else:
# Fallback: if it's a bare filename, try inside past_games
maybe = os.path.join(past_dir, arg)
if os.path.isfile(maybe):
candidate_path = maybe
if not candidate_path:
print("Usage: generate_blunder_tests.py [<game_id>|</path/to/log>]")
return 2
return _process_single_log(candidate_path)
if __name__ == "__main__":
raise SystemExit(main(sys.argv))