From 78b6c3c05356ff8448b9b87f8f6dd9ff1b76b325 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sat, 23 Aug 2025 15:53:42 +0200 Subject: [PATCH] feat: more blunder tests --- PYTHON/lichess_bot/.bot_version | 2 +- .../tests/test_blunders_OVmR29MI.py | 49 +++++++++ .../tests/test_blunders_PdZ7Ft7C.py | 62 +++++++++++ .../tests/test_blunders_VVSlhSJB.py | 42 +++++++ .../tests/test_blunders_mgh3xtEb.py | 43 ++++++++ .../tools/generate_blunder_tests.py | 103 ++++++++++++++---- 6 files changed, 280 insertions(+), 21 deletions(-) create mode 100644 PYTHON/lichess_bot/tests/test_blunders_OVmR29MI.py create mode 100644 PYTHON/lichess_bot/tests/test_blunders_PdZ7Ft7C.py create mode 100644 PYTHON/lichess_bot/tests/test_blunders_VVSlhSJB.py create mode 100644 PYTHON/lichess_bot/tests/test_blunders_mgh3xtEb.py diff --git a/PYTHON/lichess_bot/.bot_version b/PYTHON/lichess_bot/.bot_version index 8e2afd3..25bf17f 100644 --- a/PYTHON/lichess_bot/.bot_version +++ b/PYTHON/lichess_bot/.bot_version @@ -1 +1 @@ -17 \ No newline at end of file +18 \ No newline at end of file diff --git a/PYTHON/lichess_bot/tests/test_blunders_OVmR29MI.py b/PYTHON/lichess_bot/tests/test_blunders_OVmR29MI.py new file mode 100644 index 0000000..b8293b3 --- /dev/null +++ b/PYTHON/lichess_bot/tests/test_blunders_OVmR29MI.py @@ -0,0 +1,49 @@ +import os +import sys +import chess +import pytest + +# Ensure repo root is importable when running pytest directly +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +if REPO_ROOT not in sys.path: + sys.path.insert(0, REPO_ROOT) + +from PYTHON.lichess_bot.engine import RandomEngine # noqa: E402 + +BLUNDER_CASES = [ + ("rnb1kb1r/pp3ppp/4q3/2p3N1/4p3/8/PPPPNPPP/R1BQ1RK1 b kq - 1 9", "e6a2", "ply18_B_e6a2"), + ("2kr3r/1p4p1/4bp2/7p/Q2b1B2/2P5/1P3PPP/5RK1 w - - 0 20", "c3d4", "ply39_W_c3d4"), + ("2kr3r/1p4p1/4bp2/7p/Q2P1B2/8/1P3PPP/5RK1 b - - 0 20", "e6d5", "ply40_B_e6d5"), + ("2kr3r/1p4p1/5p2/3b3p/Q2P1B2/8/1P3PPP/5RK1 w - - 1 21", "a4a8", "ply41_W_a4a8"), + ("3r3r/1p1k2p1/5p2/3b3p/Q2P1B2/8/1P3PPP/5RK1 b - - 4 22", "d7c8", "ply44_B_d7c8"), + ("2kr3r/1p4p1/5p2/Q2b3p/3P1B2/8/1P3PPP/5RK1 b - - 6 23", "b7b6", "ply46_B_b7b6"), + ("2kr3r/6p1/1Q3p2/3b3p/3P1B2/8/1P3PPP/5RK1 b - - 0 24", "c8d7", "ply48_B_c8d7"), + ("3r3r/3k2p1/1Q3p2/3b3p/3P1B2/8/1P3PPP/5RK1 w - - 1 25", "f4c7", "ply49_W_f4c7"), + ("3r3r/2Bk2p1/1Q3p2/3b3p/3P4/8/1P3PPP/5RK1 b - - 2 25", "d8c8", "ply50_B_d8c8"), + ("2r4r/2Bk2p1/1Q3p2/3b3p/3P4/8/1P3PPP/5RK1 w - - 3 26", "c7b8", "ply51_W_c7b8"), + ("1r5r/Q5p1/4kp2/3b3p/3P4/8/1P3PPP/4R1K1 b - - 3 28", "e6d6", "ply56_B_e6d6"), + ("1r5r/2k1R1p1/5p2/3Q3p/3P4/8/1P3PPP/6K1 b - - 2 31", "c7c8", "ply62_B_c7c8"), + ("1rk4r/4R1p1/5p2/3Q3p/3P4/8/1P3PPP/6K1 w - - 3 32", "d5c5", "ply63_W_d5c5"), + ("1r1k3r/4R1p1/5p2/2Q4p/3P4/8/1P3PPP/6K1 w - - 5 33", "d4d5", "ply65_W_d4d5"), + ("3k3r/1Q2R3/5pp1/3P3p/8/8/1P3PPP/6K1 w - - 0 37", "d5d6", "ply73_W_d5d6"), + ("3k3r/1Q2R3/3P1p2/6pp/8/8/1P3PPP/6K1 w - - 0 38", "b7b8", "ply75_W_b7b8"), +] + +@pytest.mark.parametrize('fen,blunder_uci,label', BLUNDER_CASES, ids=[c[2] for c in BLUNDER_CASES]) +def test_engine_avoids_logged_blunder(fen, blunder_uci, label): + board = chess.Board(fen) + eng = RandomEngine(depth=4, max_time_sec=1.2) + # Prefer explanation variant if available for better failure messages + move = None + explanation = '' + if hasattr(eng, 'choose_move_with_explanation'): + try: + mv, expl = eng.choose_move_with_explanation(board, time_budget_sec=1.2) + move, explanation = mv, expl or '' + except Exception: + move = eng.choose_move(board) + else: + move = eng.choose_move(board) + assert move is not None, 'Engine returned no move' + assert move in board.legal_moves, 'Engine move is illegal' + assert move.uci() != blunder_uci, f'Engine repeated blunder {blunder_uci} at {label}. Explanation: {explanation}' diff --git a/PYTHON/lichess_bot/tests/test_blunders_PdZ7Ft7C.py b/PYTHON/lichess_bot/tests/test_blunders_PdZ7Ft7C.py new file mode 100644 index 0000000..00043ab --- /dev/null +++ b/PYTHON/lichess_bot/tests/test_blunders_PdZ7Ft7C.py @@ -0,0 +1,62 @@ +import os +import sys +import chess +import pytest + +# Ensure repo root is importable when running pytest directly +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +if REPO_ROOT not in sys.path: + sys.path.insert(0, REPO_ROOT) + +from PYTHON.lichess_bot.engine import RandomEngine # noqa: E402 + +BLUNDER_CASES = [ + ("rnb2rk1/pp2bppp/8/8/2qN4/2N5/PPP2PPP/R1BQK2R w - - 3 13", "b2b3", "ply25_W_b2b3"), + ("rn3rk1/pp2bppp/8/3Q1b2/8/1P6/2q2PPP/2B2K1R w - - 0 18", "d5b7", "ply35_W_d5b7"), + ("rn3rk1/pQ2bppp/8/5b2/8/1P6/2q2PPP/2B2K1R b - - 0 18", "c2f2", "ply36_B_c2f2"), + ("6k1/p2n1ppp/8/2b5/6b1/1P6/3K3P/4R3 b - - 0 27", "c5b4", "ply54_B_c5b4"), + ("6k1/p2n1ppp/8/8/5Kb1/1P6/7P/4b3 b - - 1 29", "e1c3", "ply58_B_e1c3"), + ("6k1/p2n1ppp/8/8/6K1/1Pb5/7P/8 b - - 0 30", "d7f6", "ply60_B_d7f6"), + ("6k1/p4ppp/5n2/5K2/8/1Pb5/7P/8 b - - 2 31", "f6d7", "ply62_B_f6d7"), + ("6k1/p2n1ppp/8/5K2/8/1Pb5/7P/8 w - - 3 32", "f5e4", "ply63_W_f5e4"), + ("6k1/p2n1ppp/8/8/4K3/1Pb5/7P/8 b - - 4 32", "d7f6", "ply64_B_d7f6"), + ("6k1/p4ppp/8/4b2n/4K3/1P5P/8/8 b - - 2 35", "e5b8", "ply70_B_e5b8"), + ("1b4k1/p4ppp/8/7n/4K3/1P5P/8/8 w - - 3 36", "e4f5", "ply71_W_e4f5"), + ("1b4k1/p4ppp/8/5K1n/8/1P5P/8/8 b - - 4 36", "b8d6", "ply72_B_b8d6"), + ("6k1/p4ppp/3b4/5K1n/8/1P5P/8/8 w - - 5 37", "f5g5", "ply73_W_f5g5"), + ("6k1/p4ppp/3b4/6Kn/8/1P5P/8/8 b - - 6 37", "h5f4", "ply74_B_h5f4"), + ("6k1/p4ppp/3b4/8/5nK1/1P5P/8/8 b - - 8 38", "h7h5", "ply76_B_h7h5"), + ("6k1/p4pp1/3b4/7p/5nK1/1P5P/8/8 w - - 0 39", "g4g3", "ply77_W_g4g3"), + ("6k1/p4pp1/3b4/7p/5n2/1P4KP/8/8 b - - 1 39", "f4e6", "ply78_B_f4e6"), + ("6k1/p4pp1/3bn3/7p/8/1P4KP/8/8 w - - 2 40", "g3h4", "ply79_W_g3h4"), + ("6k1/p4pp1/3bn3/7p/7K/1P5P/8/8 b - - 3 40", "e6c5", "ply80_B_e6c5"), + ("6k1/p4pp1/3b4/2n4p/7K/1P5P/8/8 w - - 4 41", "h4h5", "ply81_W_h4h5"), + ("6k1/p4pp1/3b4/2n4K/8/1P5P/8/8 b - - 0 41", "c5b3", "ply82_B_c5b3"), + ("6k1/p4pp1/3b4/6K1/8/1n5P/8/8 b - - 1 42", "d6e7", "ply84_B_d6e7"), + ("6k1/p3bpp1/8/5K2/8/1n5P/8/8 b - - 3 43", "b3d4", "ply86_B_b3d4"), + ("8/p3kpp1/5b2/8/3nK3/7P/8/8 w - - 10 47", "e4f4", "ply93_W_e4f4"), + ("8/p3kpp1/4nb2/8/5K2/7P/8/8 w - - 12 48", "f4f5", "ply95_W_f4f5"), + ("6k1/p4pp1/5b2/3K4/3n4/7P/8/8 w - - 18 51", "d5d6", "ply101_W_d5d6"), + ("6k1/p4pp1/4nb2/2K5/8/7P/8/8 w - - 24 54", "c5d6", "ply107_W_c5d6"), + ("6k1/p4pp1/3K1b2/8/5n2/7P/8/8 w - - 26 55", "d6d7", "ply109_W_d6d7"), + ("6k1/p2K1pp1/5b2/8/8/7n/8/8 w - - 0 56", "d7e8", "ply111_W_d7e8"), +] + +@pytest.mark.parametrize('fen,blunder_uci,label', BLUNDER_CASES, ids=[c[2] for c in BLUNDER_CASES]) +def test_engine_avoids_logged_blunder(fen, blunder_uci, label): + board = chess.Board(fen) + eng = RandomEngine(depth=4, max_time_sec=1.2) + # Prefer explanation variant if available for better failure messages + move = None + explanation = '' + if hasattr(eng, 'choose_move_with_explanation'): + try: + mv, expl = eng.choose_move_with_explanation(board, time_budget_sec=1.2) + move, explanation = mv, expl or '' + except Exception: + move = eng.choose_move(board) + else: + move = eng.choose_move(board) + assert move is not None, 'Engine returned no move' + assert move in board.legal_moves, 'Engine move is illegal' + assert move.uci() != blunder_uci, f'Engine repeated blunder {blunder_uci} at {label}. Explanation: {explanation}' diff --git a/PYTHON/lichess_bot/tests/test_blunders_VVSlhSJB.py b/PYTHON/lichess_bot/tests/test_blunders_VVSlhSJB.py new file mode 100644 index 0000000..39fc776 --- /dev/null +++ b/PYTHON/lichess_bot/tests/test_blunders_VVSlhSJB.py @@ -0,0 +1,42 @@ +import os +import sys +import chess +import pytest + +# Ensure repo root is importable when running pytest directly +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +if REPO_ROOT not in sys.path: + sys.path.insert(0, REPO_ROOT) + +from PYTHON.lichess_bot.engine import RandomEngine # noqa: E402 + +BLUNDER_CASES = [ + ("r1bqk2r/ppp2ppp/2np1n2/2b1p1B1/2B1P3/3P1N2/PPP2PPP/RN1Q1RK1 b kq - 1 6", "c5f2", "ply12_B_c5f2"), + ("r1bq1rk1/pp3ppp/3p1n2/2p1p1B1/2P1P3/2PP1N2/P5PP/RN1Q1RK1 b - - 0 11", "f6e4", "ply22_B_f6e4"), + ("3r1r2/pp3ppk/3N3p/2p1N3/4P3/2P5/P5PP/R2Q1RK1 b - - 0 17", "h7g8", "ply34_B_h7g8"), + ("3r1rk1/pp3pp1/3N3p/2p1N3/4P3/2P5/P5PP/R2Q1RK1 w - - 1 18", "e5f7", "ply35_W_e5f7"), + ("3r1rk1/pp3Np1/3N3p/2p5/4P3/2P5/P5PP/R2Q1RK1 b - - 0 18", "f8f7", "ply36_B_f8f7"), + ("5Q2/p5pk/4P2p/1pp5/8/2P5/P5PP/5RK1 b - - 0 24", "h7g6", "ply48_B_h7g6"), + ("5Q2/p5p1/4P1kp/1pp5/8/2P5/P5PP/5RK1 w - - 1 25", "e6e7", "ply49_W_e6e7"), + ("5Q2/p3P1p1/6kp/1pp5/8/2P5/P5PP/5RK1 b - - 0 25", "g6h7", "ply50_B_g6h7"), + ("4QQ2/p6k/6pp/1pp5/8/2P5/P5PP/5RK1 w - - 0 27", "f8h8", "ply53_W_f8h8"), +] + +@pytest.mark.parametrize('fen,blunder_uci,label', BLUNDER_CASES, ids=[c[2] for c in BLUNDER_CASES]) +def test_engine_avoids_logged_blunder(fen, blunder_uci, label): + board = chess.Board(fen) + eng = RandomEngine(depth=4, max_time_sec=1.2) + # Prefer explanation variant if available for better failure messages + move = None + explanation = '' + if hasattr(eng, 'choose_move_with_explanation'): + try: + mv, expl = eng.choose_move_with_explanation(board, time_budget_sec=1.2) + move, explanation = mv, expl or '' + except Exception: + move = eng.choose_move(board) + else: + move = eng.choose_move(board) + assert move is not None, 'Engine returned no move' + assert move in board.legal_moves, 'Engine move is illegal' + assert move.uci() != blunder_uci, f'Engine repeated blunder {blunder_uci} at {label}. Explanation: {explanation}' diff --git a/PYTHON/lichess_bot/tests/test_blunders_mgh3xtEb.py b/PYTHON/lichess_bot/tests/test_blunders_mgh3xtEb.py new file mode 100644 index 0000000..d710e12 --- /dev/null +++ b/PYTHON/lichess_bot/tests/test_blunders_mgh3xtEb.py @@ -0,0 +1,43 @@ +import os +import sys +import chess +import pytest + +# Ensure repo root is importable when running pytest directly +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +if REPO_ROOT not in sys.path: + sys.path.insert(0, REPO_ROOT) + +from PYTHON.lichess_bot.engine import RandomEngine # noqa: E402 + +BLUNDER_CASES = [ + ("r4r2/p2p3k/n2Np1p1/q1p5/P5Q1/8/3NKP2/8 b - - 2 24", "a5c7", "ply48_B_a5c7"), + ("r4N2/p6k/n5pq/P1pp4/6Q1/5N2/4KP2/8 b - - 0 30", "h6f8", "ply60_B_h6f8"), + ("r4q2/p6k/n5p1/P1pp4/6Q1/5N2/4KP2/8 w - - 0 31", "g4h3", "ply61_W_g4h3"), + ("r4qk1/p2Q4/n5p1/P1pp4/8/5N2/4KP2/8 w - - 4 33", "e2e3", "ply65_W_e2e3"), + ("4rqk1/p2Q4/n5p1/P1pp4/8/4KN2/5P2/8 w - - 6 34", "e3d2", "ply67_W_e3d2"), + ("4rqk1/p2Q4/n5p1/P1pp4/8/5N2/3K1P2/8 b - - 7 34", "d5d4", "ply68_B_d5d4"), + ("4rqk1/p2Q4/n5p1/P1p5/3p4/5N2/3K1P2/8 w - - 0 35", "d7a7", "ply69_W_d7a7"), + ("4r1k1/Q7/n5p1/P1p5/3p4/5q2/3K1P2/8 w - - 0 36", "d2c2", "ply71_W_d2c2"), + ("6k1/Q7/n5p1/P1p5/3p4/8/4rq2/3K4 w - - 0 38", "d1c1", "ply75_W_d1c1"), + ("6k1/Q7/n5p1/P1p5/3p4/8/4rq2/2K5 b - - 1 38", "f2e1", "ply76_B_f2e1"), +] + +@pytest.mark.parametrize('fen,blunder_uci,label', BLUNDER_CASES, ids=[c[2] for c in BLUNDER_CASES]) +def test_engine_avoids_logged_blunder(fen, blunder_uci, label): + board = chess.Board(fen) + eng = RandomEngine(depth=4, max_time_sec=1.2) + # Prefer explanation variant if available for better failure messages + move = None + explanation = '' + if hasattr(eng, 'choose_move_with_explanation'): + try: + mv, expl = eng.choose_move_with_explanation(board, time_budget_sec=1.2) + move, explanation = mv, expl or '' + except Exception: + move = eng.choose_move(board) + else: + move = eng.choose_move(board) + assert move is not None, 'Engine returned no move' + assert move in board.legal_moves, 'Engine move is illegal' + assert move.uci() != blunder_uci, f'Engine repeated blunder {blunder_uci} at {label}. Explanation: {explanation}' diff --git a/PYTHON/lichess_bot/tools/generate_blunder_tests.py b/PYTHON/lichess_bot/tools/generate_blunder_tests.py index 0836ee7..9d8892e 100644 --- a/PYTHON/lichess_bot/tools/generate_blunder_tests.py +++ b/PYTHON/lichess_bot/tools/generate_blunder_tests.py @@ -1,18 +1,32 @@ #!/usr/bin/env python3 """ -Generate pytest cases from a lichess analysis log. +Generate pytest cases from one or more lichess analysis logs. -Input: a log file that contains a "Columns:" section and a "PGN:" section, -like the example the user provided. We'll extract each row where class==Blunder, -reconstruct the FEN of the position before the blunder, and the blunder move in -UCI. Then we write a parametrized pytest that asserts the engine does not pick -that same blunder move from that position. +Input: log files that contain a "Columns:" section and a "PGN:" section. +We'll extract each row where class==Blunder, reconstruct the FEN of the +position before the blunder, and the blunder move in UCI. Then we'll write +parametrized pytest files that assert the engine does not pick that same +blunder move from those positions. -Usage: - python PYTHON/lichess_bot/tools/generate_blunder_tests.py /path/to/lichess_bot_game_xxxxx.log +Where logs are loaded from: + - By default (no arguments), all logs in the "past_games" folder located + next to this script will be processed (files matching lichess_bot_game_*.log). + - If a single argument is provided and it's a file path, that file is used. + - If a single argument looks like a game id (e.g. OVmR29MI), the script will + look for past_games/lichess_bot_game_.log next to this script. -It will create a file like: - PYTHON/lichess_bot/tests/test_blunders_.py +Usage examples: + # Process all logs in tools/past_games + python PYTHON/lichess_bot/tools/generate_blunder_tests.py + + # Process a specific game by id from tools/past_games + python PYTHON/lichess_bot/tools/generate_blunder_tests.py OVmR29MI + + # Process an explicit file path + python PYTHON/lichess_bot/tools/generate_blunder_tests.py /path/to/lichess_bot_game_xxxxx.log + +It will create files like: + PYTHON/lichess_bot/tests/test_blunders_.py Dependencies: python-chess, pytest (already in requirements.txt) """ @@ -168,30 +182,30 @@ BLUNDER_CASES = [ print(f"Wrote {target_path} with {len(cases)} blunder checks (game {game_id}).") -def main(argv: List[str]) -> int: - if len(argv) < 2: - print("Usage: generate_blunder_tests.py /path/to/lichess_bot_game_xxx.log") +def _process_single_log(log_path: str) -> int: + """Process a single log file. Returns 0 on success, non-zero otherwise.""" + try: + with open(log_path, "r", encoding="utf-8") as fh: + text = fh.read() + except FileNotFoundError: + print(f"Log file not found: {log_path}") return 2 - log_path = argv[1] - with open(log_path, "r", encoding="utf-8") as fh: - text = fh.read() blunders = parse_columns_for_blunders(text) if not blunders: - print("No blunders found in the log's Columns section.") + print(f"No blunders found in Columns section: {os.path.basename(log_path)}") return 1 pgn_text = extract_pgn(text) if not pgn_text: - print("No PGN section found in the log.") + print(f"No PGN section found: {os.path.basename(log_path)}") return 1 cases = fen_and_uci_for_blunders(pgn_text, blunders) if not cases: - print("Failed to reconstruct any blunder positions from PGN.") + print(f"Failed to reconstruct any blunder positions from PGN: {os.path.basename(log_path)}") return 1 - # Try to derive game id from file name base = os.path.basename(log_path) m = re.search(r"game_([A-Za-z0-9]+)\.log$", base) game_id = m.group(1) if m else os.path.splitext(base)[0] @@ -202,5 +216,54 @@ def main(argv: List[str]) -> int: return 0 +def main(argv: List[str]) -> int: + script_dir = os.path.dirname(__file__) + past_dir = os.path.abspath(os.path.join(script_dir, "past_games")) + + # No argument: process all logs in past_games + if len(argv) == 1: + if not os.path.isdir(past_dir): + print(f"No past_games directory found at {past_dir}") + return 2 + logs = [ + os.path.join(past_dir, name) + for name in os.listdir(past_dir) + if re.match(r"lichess_bot_game_[A-Za-z0-9]+\.log$", name) + ] + if not logs: + print(f"No logs found in {past_dir}") + return 1 + # Sort by mtime ascending for determinism + logs.sort(key=lambda p: os.path.getmtime(p)) + ok = 0 + for lp in logs: + rc = _process_single_log(lp) + if rc == 0: + ok += 1 + print(f"Processed {len(logs)} logs from {past_dir}, succeeded: {ok}, failed: {len(logs)-ok}") + return 0 if ok > 0 else 1 + + # One argument: game id or file path + arg = argv[1] + candidate_path = None + if os.path.isfile(arg): + candidate_path = arg + else: + # Treat as game id, resolve within past_games + if re.fullmatch(r"[A-Za-z0-9]+", arg): + candidate_path = os.path.join(past_dir, f"lichess_bot_game_{arg}.log") + else: + # Fallback: if it's a bare filename, try inside past_games + maybe = os.path.join(past_dir, arg) + if os.path.isfile(maybe): + candidate_path = maybe + + if not candidate_path: + print("Usage: generate_blunder_tests.py [|]") + return 2 + + return _process_single_log(candidate_path) + + if __name__ == "__main__": raise SystemExit(main(sys.argv))