From 4b156dcc9b3573707512316ee87b7cf43a23e6fb Mon Sep 17 00:00:00 2001 From: big-c-note Date: Sun, 10 May 2020 22:44:32 -0400 Subject: [PATCH 01/42] adding probability of reach for each player, probably buggy --- pluribus/games/short_deck/state.py | 278 +++++++++++++++++++++++++++++ research/test_methodology/RT.py | 51 ++++++ 2 files changed, 329 insertions(+) create mode 100644 research/test_methodology/RT.py diff --git a/pluribus/games/short_deck/state.py b/pluribus/games/short_deck/state.py index af2e08b7..89657b3c 100644 --- a/pluribus/games/short_deck/state.py +++ b/pluribus/games/short_deck/state.py @@ -7,6 +7,8 @@ import operator import os from typing import Any, Dict, List, Optional, Tuple +import joblib +from itertools import combinations import dill as pickle @@ -40,6 +42,101 @@ def new_game( return state +# TODO: there is probably a better place for agent related items +class Agent: + # TODO(fedden): Note from the supplementary material, the data here will + # need to be lower precision: "To save memory, regrets were + # stored using 4-byte integers rather than 8-byte doubles. + # There was also a floor on regret at -310,000,000 for every + # action. This made it easier to unprune actions that were + # initially pruned but later improved. This also prevented + # integer overflows". + def __init__(self): + self.strategy = collections.defaultdict( + lambda: collections.defaultdict(lambda: 0) + ) + self.regret = collections.defaultdict( + lambda: collections.defaultdict(lambda: 0) + ) + + +# TODO: Change to Leon's newest iteration on this method +class TrainedAgent(Agent): + """ + Agent who has been trained + Points to a folder whose strategies is calculated from regret and then averaged + """ + + def __init__(self, directory: str): + super().__init__() + self.offline_strategy = self._load_regret(directory) + + # TODO: the following could use a refactor, just getting through this rather quickly + def _calculate_strategy( + self, + regret: Dict[str, Dict[str, float]], + sigma: Dict[int, Dict[str, Dict[str, float]]], + I: str, + ): + """ + Get strategy from regret + """ + rsum = sum([max(x, 0) for x in regret[I].values()]) + ACTIONS = regret[I].keys() # TODO: this is hacky, might be a better way + for a in ACTIONS: + if rsum > 0: + sigma[I][a] = max(regret[I][a], 0) / rsum + else: + sigma[I][a] = 1 / len(ACTIONS) + return sigma + + def _average_strategy(self, directory: str): + files = [ + x + for x in os.listdir(directory) + if os.path.isfile(os.path.join(directory, x)) + ] + + offline_strategy = collections.defaultdict( + lambda: collections.defaultdict(lambda: 0) + ) + strategy_tmp = collections.defaultdict( + lambda: collections.defaultdict(lambda: 0) + ) + + for idx, f in enumerate(files): + if f in ["config.yaml", "strategy.gz"]: + continue + + regret_dict = joblib.load(directory + "/" + f)["regret"] + sigma = collections.defaultdict( + lambda: collections.defaultdict(lambda: 1 / 3) + ) + + for info_set, regret in sorted(regret_dict.items()): + sigma = self._calculate_strategy(regret_dict, sigma, info_set) + + for info_set, strategy in sigma.items(): + for action, probability in strategy.items(): + try: + strategy_tmp[info_set][action] += probability + except KeyError: + strategy_tmp[info_set][action] = probability + + for info_set, strategy in sorted(strategy_tmp.items()): + norm = sum(list(strategy.values())) + for action, probability in strategy.items(): + try: + offline_strategy[info_set][action] += probability / norm + except KeyError: + offline_strategy[info_set][action] = probability / norm + + return offline_strategy + + def _load_regret(self, directory: str): + return self._average_strategy(directory) + + class ShortDeckPokerState: """The state of a Short Deck Poker game at some given point in time. @@ -54,6 +151,8 @@ def __init__( big_blind: int = 100, pickle_dir: str = ".", load_pickle_files: bool = True, + real_time: bool = False, + offline_strategy: Dict = None, ): """Initialise state.""" n_players = len(players) @@ -74,6 +173,7 @@ def __init__( self._initial_n_chips = players[0].n_chips self.small_blind = small_blind self.big_blind = big_blind + self.real_time = real_time self._poker_engine = PokerEngine( table=self._table, small_blind=small_blind, big_blind=big_blind ) @@ -84,7 +184,9 @@ def __init__( self._table.dealer.deal_private_cards(self._table.players) # Store the actions as they come in here. self._history: Dict[str, List[str]] = collections.defaultdict(list) + self._public_information: Dict[str, List[Card]] = collections.defaultdict(list) self._betting_stage = "pre_flop" + self._previous_betting_stage = None self._betting_stage_to_round: Dict[str, int] = { "pre_flop": 0, "flop": 1, @@ -113,6 +215,17 @@ def __init__( player.is_turn = False self.current_player.is_turn = True + # only want to do these actions in real game play, as they are slow + if self.real_time: + # must have offline strategy loaded up + assert offline_strategy + self._offline_strategy = offline_strategy + self._starting_hand_probs = self._initialize_starting_hands() + cards_in_deck = self._table.dealer.deck._cards_in_deck + # TODO: We might not need this + self._evals = [c.eval_card for c in cards_in_deck] + self._evals_to_cards = {i.eval_card: i for i in cards_in_deck} + def __repr__(self): """Return a helpful description of object in strings and debugger.""" return f"" @@ -175,6 +288,11 @@ def apply_action(self, action_str: Optional[str]) -> ShortDeckPokerState: skip_actions = ["skip" for _ in range(new_state._skip_counter)] new_state._history[new_state.betting_stage] += skip_actions new_state._history[new_state.betting_stage].append(str(action)) + # save public information + if self._first_move_of_current_round: + new_state._public_information[ + new_state.betting_stage + ] = self._table.community_cards new_state._n_actions += 1 new_state._skip_counter = 0 # Player has made move, increment the player that is next. @@ -254,23 +372,173 @@ def _increment_stage(self): if self._betting_stage == "pre_flop": # Progress from private cards to the flop. self._betting_stage = "flop" + self._previous_betting_stage = "pre_flop" self._poker_engine.table.dealer.deal_flop(self._table) elif self._betting_stage == "flop": # Progress from flop to turn. self._betting_stage = "turn" + self._previous_betting_stage = "flop" self._poker_engine.table.dealer.deal_turn(self._table) elif self._betting_stage == "turn": # Progress from turn to river. self._betting_stage = "river" + self._previous_betting_stage = "turn" self._poker_engine.table.dealer.deal_river(self._table) elif self._betting_stage == "river": # Progress to the showdown. self._betting_stage = "show_down" + self._previous_betting_stage = "river" elif self._betting_stage in {"show_down", "terminal"}: pass else: raise ValueError(f"Unknown betting_stage: {self._betting_stage}") + def _normalize_bayes(self): + """Normalize probability of reach for each player""" + n_players = len(self.players) + for player in range(n_players): + total_prob = sum(self._starting_hand_probs[player].values()) + for starting_hand, prob in self._starting_hand_probs[player].items(): + self._starting_hand_probs[player][starting_hand] = prob / total_prob + + def update_hole_cards_bayes(self): + """Get probability of reach for each pair of hole cards for each player""" + n_players = len(self._table.players) + player_indices: List[int] = [p_i for p_i in range(n_players)] + for p_i in player_indices: + for starting_hand in self._starting_hand_probs[p_i].keys(): + # TODO: is this bad? + if "p_reach" in locals(): + del p_reach + action_sequence = [] + for idx, betting_stage in enumerate(self._history.keys()): + if self._first_move_of_current_round: + betting_stage = self._previous_betting_stage + n_actions_round = len(self._history[betting_stage]) + for i in range(n_actions_round): + action = self._history[betting_stage][i] + # TODO: maybe a method already exists for this? + if betting_stage == "pre_flop": + ph = (i + 2) % n_players + else: + ph = i % n_players + if p_i != ph: + prob_reach_all_hands = [] + num_hands = 0 + for opp_starting_hand in self._starting_hand_probs[ + p_i + ].keys(): + # TODO: clean this up + public_evals = [ + c.eval_card + for c in self._public_information[betting_stage] + ] + if len( + set(starting_hand) + .union(set(opp_starting_hand)) + .union(set(public_evals)) + ) < (len(public_evals) + 4): + prob = 0 + prob_reach_all_hands.append(prob) + else: + num_hands += 1 + public_cards = self._public_information[ + betting_stage + ] + infoset = self._info_set_helper( + opp_starting_hand, + public_cards, + action_sequence, + betting_stage, + ) + # check to see if the strategy exists, if not equal probability + # TODO: is this hacky? problem with defaulting to 1 / 3, is that it + # doesn't work for calculations that need to be made with the object's values + if self._offline_strategy[infoset].keys(): + prob = self._offline_strategy[infoset][action] + else: + prob = 1 / len(self.legal_actions) + prob_reach_all_hands.append(prob) + if "p_reach" not in locals(): + p_reach = sum(prob_reach_all_hands) / num_hands + else: + p_reach *= p_reach + elif p_i == ph: + public_evals = [ + c.eval_card + for c in self._public_information[betting_stage] + ] + if len(set(starting_hand).union(set(public_evals))) < ( + len(public_evals) + 2 + ): + prob = 0 + prob_reach_all_hands.append(prob) + else: + public_cards = self._public_information[betting_stage] + infoset = self._info_set_helper( + starting_hand, + public_cards, + action_sequence, + betting_stage, + ) + if self._offline_strategy[infoset].keys(): + prob = self._offline_strategy[infoset][action] + else: + prob = 1 / len(self.legal_actions) + if "p_reach" not in locals(): + p_reach = prob + else: + p_reach *= p_reach + action_sequence.append(action) + self._starting_hand_probs[p_i][tuple(starting_hand)] = p_reach + self._normalize_bayes() + + def _initialize_starting_hands(self): + """Dictionary of starting hands to store probabilities in""" + assert self.betting_stage == "pre_flop" + # TODO: make this abstracted for n_players + starting_hand_probs = {0: {}, 1: {}, 2: {}} + n_players = len(self.players) + starting_hands = self._get_card_combos(2) + for p_i in range(n_players): + for starting_hand in starting_hands: + starting_hand_probs[p_i][ + tuple([c.eval_card for c in starting_hand]) + ] = 1 + return starting_hand_probs + + def _info_set_helper( + self, hole_cards, public_cards, action_sequence, betting_stage + ): + # didn't want to combine this with the other, as we may want to modularize soon + """Get the information set for the current player.""" + cards = sorted(hole_cards, reverse=True,) + cards += sorted(public_cards, reverse=True,) + eval_cards = tuple(cards) + try: + cards_cluster = self.info_set_lut[betting_stage][eval_cards] + except KeyError: + if not self.info_set_lut: + raise ValueError("Pickle luts must be loaded for info set.") + elif eval_cards not in self.info_set_lut[self._betting_stage]: + raise ValueError("Cards {cards} not in pickle files.") + else: + raise ValueError("Unrecognised betting stage in pickle files.") + info_set_dict = { + "cards_cluster": cards_cluster, + "history": [ + {betting_stage: [str(action) for action in actions]} + for betting_stage, actions in self._history.items() + ], + } + return json.dumps( + info_set_dict, separators=(",", ":"), cls=utils.io.NumpyJSONEncoder + ) + + def _get_card_combos(self, num_cards): + """Get combinations of cards""" + return list(combinations(self._table.dealer.deck._cards_in_deck, num_cards)) + @property def community_cards(self) -> List[Card]: """Return all shared/public cards.""" @@ -296,6 +564,11 @@ def betting_stage(self) -> str: """Return betting stage.""" return self._betting_stage + @property + def previous_betting_stage(self) -> str: + """Return previous betting stage.""" + return self._previous_betting_stage + @property def all_players_have_actioned(self) -> bool: """Return whether all players have made atleast one action.""" @@ -306,6 +579,11 @@ def n_players_started_round(self) -> bool: """Return n_players that started the round.""" return self._n_players_started_round + @property + def first_move_of_current_round(self) -> bool: + """Return boolfor first move of current round.""" + return self._first_move_of_current_round + @property def player_i(self) -> int: """Get the index of the players turn it is.""" diff --git a/research/test_methodology/RT.py b/research/test_methodology/RT.py new file mode 100644 index 00000000..bb615000 --- /dev/null +++ b/research/test_methodology/RT.py @@ -0,0 +1,51 @@ +from typing import Dict + +from pluribus.games.short_deck.state import * + + +# this is a decent function for loading up a particular action sequence +def get_game_state(state: ShortDeckPokerState, action_sequence: list): + """Follow through the action sequence provided to get current node""" + if not action_sequence: + return state + a = action_sequence.pop(0) + state.apply_action(a) + if a == "skip": + a = action_sequence.pop(0) + state.apply_action(a) + new_state = state.apply_action(a) + return get_game_state(new_state, action_sequence) + + +# added some flags for RT +def new_rt_game( + n_players: int, offline_strategy: Dict, real_time=True +) -> ShortDeckPokerState: + """Create a new game of short deck poker.""" + pot = Pot() + players = [ + ShortDeckPokerPlayer(player_i=player_i, initial_chips=10000, pot=pot) + for player_i in range(n_players) + ] + state = ShortDeckPokerState( + players=players, offline_strategy=offline_strategy, real_time=real_time + ) + return state + + +if __name__ == "__main__": + # we load a (trained) strategy + agent1 = TrainedAgent("../blueprint_algo/results_2020_05_10_21_36_47_291425") + action_sequence = ["raise", "call", "raise"] + state: ShortDeckPokerState = new_rt_game(3, agent1.offline_strategy) + # load up a particular action sequence + current_game_state = get_game_state(state, action_sequence) + # decided to make this a one time method rather than something that always updates + # reason being: we won't need it except for a few choice nodes + current_game_state.update_hole_cards_bayes() + + import ipdb + ipdb.set_trace() + # TODO: need function for dealing starting hands according to the bayesian updates + # TODO: need a function to load a particular flop + # TODO: use that game state and run CFR to update strategy (easy as passing that state to cfr) From 3b2cc02735f2d2e1acefa97e2c5b3546507e146b Mon Sep 17 00:00:00 2001 From: big-c-note Date: Mon, 11 May 2020 15:23:54 -0400 Subject: [PATCH 02/42] adding method for dealing cards, changing real time flag since these are test-specific changes atm --- pluribus/games/short_deck/state.py | 40 +++++++++++++++++++++++++++--- research/test_methodology/RT.py | 8 +++--- 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/pluribus/games/short_deck/state.py b/pluribus/games/short_deck/state.py index 89657b3c..e87b9244 100644 --- a/pluribus/games/short_deck/state.py +++ b/pluribus/games/short_deck/state.py @@ -9,8 +9,10 @@ from typing import Any, Dict, List, Optional, Tuple import joblib from itertools import combinations +import random import dill as pickle +import numpy as np from pluribus import utils from pluribus.poker.card import Card @@ -151,7 +153,7 @@ def __init__( big_blind: int = 100, pickle_dir: str = ".", load_pickle_files: bool = True, - real_time: bool = False, + real_time_test: bool = False, offline_strategy: Dict = None, ): """Initialise state.""" @@ -173,7 +175,7 @@ def __init__( self._initial_n_chips = players[0].n_chips self.small_blind = small_blind self.big_blind = big_blind - self.real_time = real_time + self.real_time_test = real_time_test self._poker_engine = PokerEngine( table=self._table, small_blind=small_blind, big_blind=big_blind ) @@ -181,7 +183,8 @@ def __init__( # this), assign blinds to the players. self._poker_engine.round_setup() # Deal private cards to players. - self._table.dealer.deal_private_cards(self._table.players) + if not self.real_time_test: + self._table.dealer.deal_private_cards(self._table.players) # Store the actions as they come in here. self._history: Dict[str, List[str]] = collections.defaultdict(list) self._public_information: Dict[str, List[Card]] = collections.defaultdict(list) @@ -216,7 +219,7 @@ def __init__( self.current_player.is_turn = True # only want to do these actions in real game play, as they are slow - if self.real_time: + if self.real_time_test: # must have offline strategy loaded up assert offline_strategy self._offline_strategy = offline_strategy @@ -493,6 +496,35 @@ def update_hole_cards_bayes(self): self._starting_hand_probs[p_i][tuple(starting_hand)] = p_reach self._normalize_bayes() + def deal_bayes(self): + players = list(range(len(self.players))) + random.shuffle(players) + cards_selected = [] + + for player in players: + # does this maintain order? + starting_hand_eval = self._get_starting_hand(player) + len_union = len(set(starting_hand_eval).union(set(cards_selected))) + len_individual = len(starting_hand_eval) + len(cards_selected) + while len_union < len_individual: + starting_hand_eval = self._get_starting_hand(player) + len_union = len(set(starting_hand_eval).union(set(cards_selected))) + len_individual = len(starting_hand_eval) + len(cards_selected) + for card_eval in starting_hand_eval: + card = self._evals_to_cards[card_eval] + self.players[player].add_private_card(card) + cards_selected += starting_hand_eval + + def _get_starting_hand(self, player_idx: int): + """Get starting hand based on probability of reach""" + starting_hand_idxs = list(range(len(self._starting_hand_probs[player_idx].keys()))) + starting_hands_probs = list(self._starting_hand_probs[player_idx].values()) + starting_hand_idx = np.random.choice(starting_hand_idxs, 1, p=starting_hands_probs)[0] + import ipdb; + ipdb.set_trace() + starting_hand = list(self._starting_hand_probs[player_idx].keys())[starting_hand_idx] + return starting_hand + def _initialize_starting_hands(self): """Dictionary of starting hands to store probabilities in""" assert self.betting_stage == "pre_flop" diff --git a/research/test_methodology/RT.py b/research/test_methodology/RT.py index bb615000..812b88aa 100644 --- a/research/test_methodology/RT.py +++ b/research/test_methodology/RT.py @@ -19,7 +19,7 @@ def get_game_state(state: ShortDeckPokerState, action_sequence: list): # added some flags for RT def new_rt_game( - n_players: int, offline_strategy: Dict, real_time=True + n_players: int, offline_strategy: Dict, real_time_test=True ) -> ShortDeckPokerState: """Create a new game of short deck poker.""" pot = Pot() @@ -28,7 +28,7 @@ def new_rt_game( for player_i in range(n_players) ] state = ShortDeckPokerState( - players=players, offline_strategy=offline_strategy, real_time=real_time + players=players, offline_strategy=offline_strategy, real_time_test=real_time_test ) return state @@ -43,7 +43,9 @@ def new_rt_game( # decided to make this a one time method rather than something that always updates # reason being: we won't need it except for a few choice nodes current_game_state.update_hole_cards_bayes() - + import ipdb + ipdb.set_trace() + current_game_state.deal_bayes() import ipdb ipdb.set_trace() # TODO: need function for dealing starting hands according to the bayesian updates From e15c405a33c8b0b7ff42726765696e86b9abac7b Mon Sep 17 00:00:00 2001 From: big-c-note Date: Mon, 11 May 2020 16:25:53 -0400 Subject: [PATCH 03/42] adding TODOs --- pluribus/games/short_deck/state.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pluribus/games/short_deck/state.py b/pluribus/games/short_deck/state.py index e87b9244..9ddca249 100644 --- a/pluribus/games/short_deck/state.py +++ b/pluribus/games/short_deck/state.py @@ -217,7 +217,7 @@ def __init__( for player in self.players: player.is_turn = False self.current_player.is_turn = True - + # TODO add attribute of public_cards, that can be supplied by convenience method # only want to do these actions in real game play, as they are slow if self.real_time_test: # must have offline strategy loaded up @@ -376,16 +376,19 @@ def _increment_stage(self): # Progress from private cards to the flop. self._betting_stage = "flop" self._previous_betting_stage = "pre_flop" + # TODO check to see if there are supplied public cards, and then use those self._poker_engine.table.dealer.deal_flop(self._table) elif self._betting_stage == "flop": # Progress from flop to turn. self._betting_stage = "turn" self._previous_betting_stage = "flop" + # TODO check to see if there are supplied public cards, and then use those self._poker_engine.table.dealer.deal_turn(self._table) elif self._betting_stage == "turn": # Progress from turn to river. self._betting_stage = "river" self._previous_betting_stage = "turn" + # TODO check to see if there are supplied public cards, and then use those self._poker_engine.table.dealer.deal_river(self._table) elif self._betting_stage == "river": # Progress to the showdown. @@ -499,6 +502,8 @@ def update_hole_cards_bayes(self): def deal_bayes(self): players = list(range(len(self.players))) random.shuffle(players) + + # TODO should contain the current public cards/heros real hand, if exists cards_selected = [] for player in players: @@ -515,13 +520,13 @@ def deal_bayes(self): self.players[player].add_private_card(card) cards_selected += starting_hand_eval + # TODO add convenience method to supply public cards + def _get_starting_hand(self, player_idx: int): """Get starting hand based on probability of reach""" starting_hand_idxs = list(range(len(self._starting_hand_probs[player_idx].keys()))) starting_hands_probs = list(self._starting_hand_probs[player_idx].values()) starting_hand_idx = np.random.choice(starting_hand_idxs, 1, p=starting_hands_probs)[0] - import ipdb; - ipdb.set_trace() starting_hand = list(self._starting_hand_probs[player_idx].keys())[starting_hand_idx] return starting_hand From cb91e0f0e0635c01507a3e6ae2d15892a7451c40 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Mon, 11 May 2020 19:52:10 -0400 Subject: [PATCH 04/42] fix infoset lookup issue --- pluribus/games/short_deck/state.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/pluribus/games/short_deck/state.py b/pluribus/games/short_deck/state.py index 9ddca249..ce32aa8a 100644 --- a/pluribus/games/short_deck/state.py +++ b/pluribus/games/short_deck/state.py @@ -155,6 +155,7 @@ def __init__( load_pickle_files: bool = True, real_time_test: bool = False, offline_strategy: Dict = None, + public_cards: List[Card] = None ): """Initialise state.""" n_players = len(players) @@ -214,10 +215,15 @@ def __init__( self._skip_counter = 0 self._first_move_of_current_round = True self._reset_betting_round_state() +<<<<<<< HEAD for player in self.players: player.is_turn = False self.current_player.is_turn = True # TODO add attribute of public_cards, that can be supplied by convenience method +======= + self._public_cards = public_cards + +>>>>>>> 9f665cd... fix infoset lookup issue # only want to do these actions in real game play, as they are slow if self.real_time_test: # must have offline strategy loaded up @@ -407,6 +413,13 @@ def _normalize_bayes(self): for starting_hand, prob in self._starting_hand_probs[player].items(): self._starting_hand_probs[player][starting_hand] = prob / total_prob + # def set_public_cards(self, public_cards): + # """Method to add public cards for RTS""" + # # can't add them if you have community cards already + # assert not self._public_cards + # assert not self._table.community_cards + # self._public_cards = public_cards + def update_hole_cards_bayes(self): """Get probability of reach for each pair of hole cards for each player""" n_players = len(self._table.players) @@ -451,9 +464,10 @@ def update_hole_cards_bayes(self): public_cards = self._public_information[ betting_stage ] + public_cards_evals = [c.eval_card for c in public_cards] infoset = self._info_set_helper( opp_starting_hand, - public_cards, + public_cards_evals, action_sequence, betting_stage, ) @@ -481,9 +495,10 @@ def update_hole_cards_bayes(self): prob_reach_all_hands.append(prob) else: public_cards = self._public_information[betting_stage] + public_cards_evals = [c.eval_card for c in public_cards] infoset = self._info_set_helper( starting_hand, - public_cards, + public_cards_evals, action_sequence, betting_stage, ) From 9355e2bd579a57431db174046778c4b1a00c6679 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Mon, 11 May 2020 20:15:09 -0400 Subject: [PATCH 05/42] simple way of dealing with predetermined public cards --- pluribus/games/short_deck/state.py | 31 +++++++++++++++++------------- research/test_methodology/RT.py | 7 +++++-- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/pluribus/games/short_deck/state.py b/pluribus/games/short_deck/state.py index ce32aa8a..93547089 100644 --- a/pluribus/games/short_deck/state.py +++ b/pluribus/games/short_deck/state.py @@ -215,15 +215,14 @@ def __init__( self._skip_counter = 0 self._first_move_of_current_round = True self._reset_betting_round_state() -<<<<<<< HEAD for player in self.players: player.is_turn = False self.current_player.is_turn = True # TODO add attribute of public_cards, that can be supplied by convenience method -======= self._public_cards = public_cards - ->>>>>>> 9f665cd... fix infoset lookup issue + if public_cards: + assert len(public_cards) in {3, 4, 5} + self._public_cards = public_cards # only want to do these actions in real game play, as they are slow if self.real_time_test: # must have offline strategy loaded up @@ -382,19 +381,32 @@ def _increment_stage(self): # Progress from private cards to the flop. self._betting_stage = "flop" self._previous_betting_stage = "pre_flop" + if self._public_cards: + community_cards = [] + for _ in range(3): + community_cards.append(self._public_cards.pop(0)) + self._poker_engine.table.community_cards += community_cards # TODO check to see if there are supplied public cards, and then use those - self._poker_engine.table.dealer.deal_flop(self._table) + else: + self._poker_engine.table.dealer.deal_flop(self._table) elif self._betting_stage == "flop": # Progress from flop to turn. self._betting_stage = "turn" self._previous_betting_stage = "flop" # TODO check to see if there are supplied public cards, and then use those - self._poker_engine.table.dealer.deal_turn(self._table) + if self._public_cards: + turn_card = self._public_cards.pop(0) + self._poker_engine.table.community_cards.append(turn_card) + else: + self._poker_engine.table.dealer.deal_turn(self._table) elif self._betting_stage == "turn": # Progress from turn to river. self._betting_stage = "river" self._previous_betting_stage = "turn" # TODO check to see if there are supplied public cards, and then use those + if self._public_cards: + river_card = self._public_cards.pop(0) + self._poker_engine.table.community_cards.append(river_card) self._poker_engine.table.dealer.deal_river(self._table) elif self._betting_stage == "river": # Progress to the showdown. @@ -413,13 +425,6 @@ def _normalize_bayes(self): for starting_hand, prob in self._starting_hand_probs[player].items(): self._starting_hand_probs[player][starting_hand] = prob / total_prob - # def set_public_cards(self, public_cards): - # """Method to add public cards for RTS""" - # # can't add them if you have community cards already - # assert not self._public_cards - # assert not self._table.community_cards - # self._public_cards = public_cards - def update_hole_cards_bayes(self): """Get probability of reach for each pair of hole cards for each player""" n_players = len(self._table.players) diff --git a/research/test_methodology/RT.py b/research/test_methodology/RT.py index 812b88aa..0aeade5a 100644 --- a/research/test_methodology/RT.py +++ b/research/test_methodology/RT.py @@ -1,8 +1,11 @@ from typing import Dict from pluribus.games.short_deck.state import * +from pluribus.poker.card import Card +public_cards = [Card("ace", "spades"), Card("jack", "spades"), Card("queen", "hearts")] + # this is a decent function for loading up a particular action sequence def get_game_state(state: ShortDeckPokerState, action_sequence: list): """Follow through the action sequence provided to get current node""" @@ -28,7 +31,7 @@ def new_rt_game( for player_i in range(n_players) ] state = ShortDeckPokerState( - players=players, offline_strategy=offline_strategy, real_time_test=real_time_test + players=players, offline_strategy=offline_strategy, real_time_test=real_time_test, public_cards=public_cards ) return state @@ -36,7 +39,7 @@ def new_rt_game( if __name__ == "__main__": # we load a (trained) strategy agent1 = TrainedAgent("../blueprint_algo/results_2020_05_10_21_36_47_291425") - action_sequence = ["raise", "call", "raise"] + action_sequence = ["raise", "call", "raise", "fold", "call", "call"] state: ShortDeckPokerState = new_rt_game(3, agent1.offline_strategy) # load up a particular action sequence current_game_state = get_game_state(state, action_sequence) From 28e3f585c8d7b5ea9e000dc843743eba5eef7110 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Mon, 11 May 2020 20:25:12 -0400 Subject: [PATCH 06/42] fixing broken tests --- pluribus/games/short_deck/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pluribus/games/short_deck/state.py b/pluribus/games/short_deck/state.py index 93547089..6ac2e018 100644 --- a/pluribus/games/short_deck/state.py +++ b/pluribus/games/short_deck/state.py @@ -222,7 +222,7 @@ def __init__( self._public_cards = public_cards if public_cards: assert len(public_cards) in {3, 4, 5} - self._public_cards = public_cards + self._public_cards = public_cards # only want to do these actions in real game play, as they are slow if self.real_time_test: # must have offline strategy loaded up From 21da4bbc0575ca86da858622f807e284b4f5bd56 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Tue, 12 May 2020 00:15:50 -0400 Subject: [PATCH 07/42] getting closer to using realtime, but I broke something in the bayes update method --- pluribus/games/short_deck/state.py | 67 ++++++---- pluribus/poker/deck.py | 8 ++ research/test_methodology/RT.py | 50 +------ research/test_methodology/RT_cfr.py | 197 ++++++++++++++++++++++++++++ 4 files changed, 252 insertions(+), 70 deletions(-) create mode 100644 research/test_methodology/RT_cfr.py diff --git a/pluribus/games/short_deck/state.py b/pluribus/games/short_deck/state.py index 6ac2e018..f694f478 100644 --- a/pluribus/games/short_deck/state.py +++ b/pluribus/games/short_deck/state.py @@ -185,7 +185,7 @@ def __init__( self._poker_engine.round_setup() # Deal private cards to players. if not self.real_time_test: - self._table.dealer.deal_private_cards(self._table.players) + self._poker_engine.table.dealer.deal_private_cards(self._table.players) # Store the actions as they come in here. self._history: Dict[str, List[str]] = collections.defaultdict(list) self._public_information: Dict[str, List[Card]] = collections.defaultdict(list) @@ -229,8 +229,8 @@ def __init__( assert offline_strategy self._offline_strategy = offline_strategy self._starting_hand_probs = self._initialize_starting_hands() - cards_in_deck = self._table.dealer.deck._cards_in_deck # TODO: We might not need this + cards_in_deck = self._table.dealer.deck._cards_in_deck self._evals = [c.eval_card for c in cards_in_deck] self._evals_to_cards = {i.eval_card: i for i in cards_in_deck} @@ -328,7 +328,11 @@ def apply_action(self, action_str: Optional[str]) -> ShortDeckPokerState: # Now check if the game is terminal. if new_state._betting_stage in {"terminal", "show_down"}: # Distribute winnings. - new_state._poker_engine.compute_winners() + try: + new_state._poker_engine.compute_winners() + except: + import ipdb; + ipdb.set_trace() break for player in new_state.players: player.is_turn = False @@ -381,32 +385,30 @@ def _increment_stage(self): # Progress from private cards to the flop. self._betting_stage = "flop" self._previous_betting_stage = "pre_flop" - if self._public_cards: - community_cards = [] - for _ in range(3): - community_cards.append(self._public_cards.pop(0)) - self._poker_engine.table.community_cards += community_cards - # TODO check to see if there are supplied public cards, and then use those + if self._public_cards + if len(self._public_cards) == 3: + community_cards = self._public_cards[:3] + self._poker_engine.table.community_cards += community_cards else: self._poker_engine.table.dealer.deal_flop(self._table) elif self._betting_stage == "flop": # Progress from flop to turn. self._betting_stage = "turn" self._previous_betting_stage = "flop" - # TODO check to see if there are supplied public cards, and then use those if self._public_cards: - turn_card = self._public_cards.pop(0) - self._poker_engine.table.community_cards.append(turn_card) + if len(self._public_cards) == 4: + community_cards = self._public_cards[3:4] + self._poker_engine.table.community_cards += community_cards else: self._poker_engine.table.dealer.deal_turn(self._table) elif self._betting_stage == "turn": # Progress from turn to river. self._betting_stage = "river" self._previous_betting_stage = "turn" - # TODO check to see if there are supplied public cards, and then use those if self._public_cards: - river_card = self._public_cards.pop(0) - self._poker_engine.table.community_cards.append(river_card) + if len(self._public_cards) == 5: + community_cards = self._public_cards[4:] + self._poker_engine.table.community_cards += community_cards self._poker_engine.table.dealer.deal_river(self._table) elif self._betting_stage == "river": # Progress to the showdown. @@ -466,6 +468,7 @@ def update_hole_cards_bayes(self): prob_reach_all_hands.append(prob) else: num_hands += 1 + public_cards = self._public_information[ betting_stage ] @@ -520,25 +523,37 @@ def update_hole_cards_bayes(self): self._normalize_bayes() def deal_bayes(self): + lut = self.info_set_lut + self.info_set_lut = {} + new_state = copy.deepcopy(self) + new_state.info_set_lut = self.info_set_lut = lut + players = list(range(len(self.players))) random.shuffle(players) # TODO should contain the current public cards/heros real hand, if exists - cards_selected = [] + card_evals_selected = [] for player in players: # does this maintain order? - starting_hand_eval = self._get_starting_hand(player) - len_union = len(set(starting_hand_eval).union(set(cards_selected))) - len_individual = len(starting_hand_eval) + len(cards_selected) + starting_hand_eval = new_state._get_starting_hand(player) + len_union = len(set(starting_hand_eval).union(set(card_evals_selected))) + len_individual = len(starting_hand_eval) + len(card_evals_selected) while len_union < len_individual: - starting_hand_eval = self._get_starting_hand(player) - len_union = len(set(starting_hand_eval).union(set(cards_selected))) - len_individual = len(starting_hand_eval) + len(cards_selected) + starting_hand_eval = new_state._get_starting_hand(player) + len_union = len(set(starting_hand_eval).union(set(card_evals_selected))) + len_individual = len(starting_hand_eval) + len(card_evals_selected) for card_eval in starting_hand_eval: - card = self._evals_to_cards[card_eval] - self.players[player].add_private_card(card) - cards_selected += starting_hand_eval + card = new_state._evals_to_cards[card_eval] + new_state.players[player].add_private_card(card) + card_evals_selected += starting_hand_eval + cards_selected = [new_state._evals_to_cards[c] for c in card_evals_selected] + cards_selected += new_state._public_cards + for card in cards_selected: + new_state._table.dealer.deck.remove(card) + import ipdb; + ipdb.set_trace() + return new_state # TODO add convenience method to supply public cards @@ -594,7 +609,7 @@ def _info_set_helper( def _get_card_combos(self, num_cards): """Get combinations of cards""" - return list(combinations(self._table.dealer.deck._cards_in_deck, num_cards)) + return list(combinations(self._poker_engine.table.dealer.deck._cards_in_deck, num_cards)) @property def community_cards(self) -> List[Card]: diff --git a/pluribus/poker/deck.py b/pluribus/poker/deck.py index efc7e5f6..4901019d 100644 --- a/pluribus/poker/deck.py +++ b/pluribus/poker/deck.py @@ -61,3 +61,11 @@ def pick(self, random: bool = True) -> Card: card: Card = self._cards_in_deck.pop(index) self._dealt_cards.append(card) return card + + def remove(self, card): + """Remove a specific card from the deck""" + # TODO: is there any reason for them to be? + # Maybe better to assert it's not + if card not in self._cards_in_deck: + self._cards_in_deck.remove(card) + self._dealt_cards.append(card) diff --git a/research/test_methodology/RT.py b/research/test_methodology/RT.py index 0aeade5a..dbbaa804 100644 --- a/research/test_methodology/RT.py +++ b/research/test_methodology/RT.py @@ -1,56 +1,18 @@ from typing import Dict +from RT_cfr import * from pluribus.games.short_deck.state import * from pluribus.poker.card import Card -public_cards = [Card("ace", "spades"), Card("jack", "spades"), Card("queen", "hearts")] - -# this is a decent function for loading up a particular action sequence -def get_game_state(state: ShortDeckPokerState, action_sequence: list): - """Follow through the action sequence provided to get current node""" - if not action_sequence: - return state - a = action_sequence.pop(0) - state.apply_action(a) - if a == "skip": - a = action_sequence.pop(0) - state.apply_action(a) - new_state = state.apply_action(a) - return get_game_state(new_state, action_sequence) - - -# added some flags for RT -def new_rt_game( - n_players: int, offline_strategy: Dict, real_time_test=True -) -> ShortDeckPokerState: - """Create a new game of short deck poker.""" - pot = Pot() - players = [ - ShortDeckPokerPlayer(player_i=player_i, initial_chips=10000, pot=pot) - for player_i in range(n_players) - ] - state = ShortDeckPokerState( - players=players, offline_strategy=offline_strategy, real_time_test=real_time_test, public_cards=public_cards - ) - return state - - if __name__ == "__main__": + utils.random.seed(38) + public_cards = [Card("ace", "spades"), Card("jack", "spades"), Card("queen", "hearts")] # we load a (trained) strategy agent1 = TrainedAgent("../blueprint_algo/results_2020_05_10_21_36_47_291425") - action_sequence = ["raise", "call", "raise", "fold", "call", "call"] - state: ShortDeckPokerState = new_rt_game(3, agent1.offline_strategy) - # load up a particular action sequence - current_game_state = get_game_state(state, action_sequence) - # decided to make this a one time method rather than something that always updates - # reason being: we won't need it except for a few choice nodes - current_game_state.update_hole_cards_bayes() + action_sequence = ["raise", "call", "raise"] import ipdb ipdb.set_trace() - current_game_state.deal_bayes() - import ipdb + agent_output = train(agent1.offline_strategy, public_cards, action_sequence, 100, 20, 20, 3) + import ipdb; ipdb.set_trace() - # TODO: need function for dealing starting hands according to the bayesian updates - # TODO: need a function to load a particular flop - # TODO: use that game state and run CFR to update strategy (easy as passing that state to cfr) diff --git a/research/test_methodology/RT_cfr.py b/research/test_methodology/RT_cfr.py new file mode 100644 index 00000000..34346f55 --- /dev/null +++ b/research/test_methodology/RT_cfr.py @@ -0,0 +1,197 @@ +""" +""" +from __future__ import annotations + +import logging + +logging.basicConfig(filename="test.txt", level=logging.DEBUG) + +from tqdm import trange + +from pluribus.games.short_deck.state import * + + +def calculate_strategy( + regret: Dict[str, Dict[str, float]], I: str, state: ShortDeckPokerState, +): + """ + + :param regret: dictionary of regrets, I is key, then each action at I, with values being regret + :param sigma: dictionary of strategy updated by regret, iteration is key, then I is key, then each action with prob + :param I: + :param state: the game state + :return: doesn't return anything, just updates sigma + """ + sigma = collections.defaultdict(lambda: collections.defaultdict(lambda: 1 / 3)) + rsum = sum([max(x, 0) for x in regret[I].values()]) + for a in state.legal_actions: + if rsum > 0: + sigma[I][a] = max(regret[I][a], 0) / rsum + else: + sigma[I][a] = 1 / len(state.legal_actions) + return sigma + + +def cfr(agent: Agent, state: ShortDeckPokerState, i: int, t: int) -> float: + """ + regular cfr algo + + :param state: the game state + :param i: player + :param t: iteration + :return: expected value for node for player i + """ + logging.debug("CFR") + logging.debug("########") + logging.debug(f"Iteration: {t}") + logging.debug(f"Player Set to Update Regret: {i}") + logging.debug(f"P(h): {state.player_i}") + logging.debug(f"P(h) Updating Regret? {state.player_i == i}") + logging.debug(f"Betting Round {state._betting_stage}") + logging.debug(f"Community Cards {state._table.community_cards}") + logging.debug(f"Player 0 hole cards: {state.players[0].cards}") + logging.debug(f"Player 1 hole cards: {state.players[1].cards}") + logging.debug(f"Player 2 hole cards: {state.players[2].cards}") + logging.debug(f"Betting Action Correct?: {state.players}") + + ph = state.player_i + + player_not_in_hand = not state.players[i].is_active + if state.is_terminal or player_not_in_hand: + return state.payout[i] + + # NOTE(fedden): The logic in Algorithm 1 in the supplementary material + # instructs the following lines of logic, but state class + # will already skip to the next in-hand player. + # elif p_i not in hand: + # cfr() + # NOTE(fedden): According to Algorithm 1 in the supplementary material, + # we would add in the following bit of logic. However we + # already have the game logic embedded in the state class, + # and this accounts for the chance samplings. In other words, + # it makes sure that chance actions such as dealing cards + # happen at the appropriate times. + # elif h is chance_node: + # sample action from strategy for h + # cfr() + + elif ph == i: + I = state.info_set + # calculate strategy + logging.debug(f"About to Calculate Strategy, Regret: {agent.regret[I]}") + logging.debug(f"Current regret: {agent.regret[I]}") + sigma = calculate_strategy(agent.regret, I, state) + logging.debug(f"Calculated Strategy for {I}: {sigma[I]}") + + vo = 0.0 + voa = {} + for a in state.legal_actions: + logging.debug( + f"ACTION TRAVERSED FOR REGRET: ph {state.player_i} ACTION: {a}" + ) + new_state: ShortDeckPokerState = state.apply_action(a) + voa[a] = cfr(agent, new_state, i, t) + logging.debug(f"Got EV for {a}: {voa[a]}") + vo += sigma[I][a] * voa[a] + logging.debug( + f"""Added to Node EV for ACTION: {a} INFOSET: {I} + STRATEGY: {sigma[I][a]}: {sigma[I][a] * voa[a]}""" + ) + logging.debug(f"Updated EV at {I}: {vo}") + + for a in state.legal_actions: + agent.regret[I][a] += voa[a] - vo + logging.debug(f"Updated Regret at {I}: {agent.regret[I]}") + + return vo + else: + # import ipdb; + # ipdb.set_trace() + Iph = state.info_set + logging.debug(f"About to Calculate Strategy, Regret: {agent.regret[Iph]}") + logging.debug(f"Current regret: {agent.regret[Iph]}") + sigma = calculate_strategy(agent.regret, Iph, state) + logging.debug(f"Calculated Strategy for {Iph}: {sigma[Iph]}") + + try: + a = np.random.choice( + list(sigma[Iph].keys()), 1, p=list(sigma[Iph].values()), + )[0] + logging.debug(f"ACTION SAMPLED: ph {state.player_i} ACTION: {a}") + + except ValueError: + p = 1 / len(state.legal_actions) + probabilities = np.full(len(state.legal_actions), p) + a = np.random.choice(state.legal_actions, p=probabilities) + sigma[Iph] = {action: p for action in state.legal_actions} + logging.debug(f"ACTION SAMPLED: ph {state.player_i} ACTION: {a}") + + new_state: ShortDeckPokerState = state.apply_action(a) + return cfr(agent, new_state, i, t) + + +# this is a decent function for loading up a particular action sequence +def get_game_state(state: ShortDeckPokerState, action_sequence: list): + """Follow through the action sequence provided to get current node""" + if not action_sequence: + return state + a = action_sequence.pop(0) + state.apply_action(a) + if a == "skip": + a = action_sequence.pop(0) + state.apply_action(a) + new_state = state.apply_action(a) + return get_game_state(new_state, action_sequence) + + +# added some flags for RT +def new_rt_game( + n_players: int, offline_strategy: Dict, action_sequence, public_cards, real_time_test=True +) -> ShortDeckPokerState: + """Create a new game of short deck poker.""" + pot = Pot() + players = [ + ShortDeckPokerPlayer(player_i=player_i, initial_chips=10000, pot=pot) + for player_i in range(n_players) + ] + state = ShortDeckPokerState( + players=players, offline_strategy=offline_strategy, real_time_test=real_time_test, public_cards=public_cards + ) + current_game_state = get_game_state(state, action_sequence) + # decided to make this a one time method rather than something that always updates + # reason being: we won't need it except for a few choice nodes + current_game_state.update_hole_cards_bayes() + return current_game_state + + +def train( + offline_strategy: Dict, + public_cards, + action_sequence: list, + n_iterations: int, + lcfr_threshold: int, + discount_interval: int, + n_players: int, +): + """Train agent.""" + utils.random.seed(38) + agent = Agent() + + current_game_state = new_rt_game(3, offline_strategy, action_sequence, public_cards) + for t in trange(1, n_iterations + 1, desc="train iter"): + if t == 2: + logging.disable(logging.DEBUG) + for i in range(n_players): # fixed position i + # Create a new state. + state: ShortDeckPokerState = current_game_state.deal_bayes() + import ipdb; + ipdb.set_trace() + cfr(agent, state, i, t) + if t < lcfr_threshold & t % discount_interval == 0: + d = (t / discount_interval) / ((t / discount_interval) + 1) + for I in agent.regret.keys(): + for a in agent.regret[I].keys(): + agent.regret[I][a] *= d + agent.strategy[I][a] *= d + + return agent From e326629be30cf9db04804c7d64f832f575599d03 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Tue, 12 May 2020 00:20:40 -0400 Subject: [PATCH 08/42] fix sytax error --- pluribus/games/short_deck/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pluribus/games/short_deck/state.py b/pluribus/games/short_deck/state.py index f694f478..b01f5a8e 100644 --- a/pluribus/games/short_deck/state.py +++ b/pluribus/games/short_deck/state.py @@ -385,7 +385,7 @@ def _increment_stage(self): # Progress from private cards to the flop. self._betting_stage = "flop" self._previous_betting_stage = "pre_flop" - if self._public_cards + if self._public_cards: if len(self._public_cards) == 3: community_cards = self._public_cards[:3] self._poker_engine.table.community_cards += community_cards From e362924e2694985ddb29806a916c928300c90797 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Tue, 12 May 2020 18:33:03 -0400 Subject: [PATCH 09/42] work around for awkward deck class behavior --- pluribus/games/short_deck/state.py | 67 +++++++++++++++++------------ pluribus/poker/deck.py | 8 ++-- research/test_methodology/RT.py | 4 +- research/test_methodology/RT_cfr.py | 20 ++++++--- 4 files changed, 60 insertions(+), 39 deletions(-) diff --git a/pluribus/games/short_deck/state.py b/pluribus/games/short_deck/state.py index b01f5a8e..87335ae9 100644 --- a/pluribus/games/short_deck/state.py +++ b/pluribus/games/short_deck/state.py @@ -213,7 +213,7 @@ def __init__( "terminal": player_i_order, } self._skip_counter = 0 - self._first_move_of_current_round = True + # self._first_move_of_current_round = True self._reset_betting_round_state() for player in self.players: player.is_turn = False @@ -266,7 +266,7 @@ def apply_action(self, action_str: Optional[str]) -> ShortDeckPokerState: new_state.info_set_lut = self.info_set_lut = lut # An action has been made, so alas we are not in the first move of the # current betting round. - new_state._first_move_of_current_round = False + # new_state._first_move_of_current_round = False if action_str is None: # Assert active player has folded already. assert ( @@ -297,10 +297,10 @@ def apply_action(self, action_str: Optional[str]) -> ShortDeckPokerState: new_state._history[new_state.betting_stage] += skip_actions new_state._history[new_state.betting_stage].append(str(action)) # save public information - if self._first_move_of_current_round: - new_state._public_information[ - new_state.betting_stage - ] = self._table.community_cards + # if new_state._first_move_of_current_round: + # new_state._public_information[ + # new_state.betting_stage + # ] = new_state._table.community_cards new_state._n_actions += 1 new_state._skip_counter = 0 # Player has made move, increment the player that is next. @@ -315,7 +315,7 @@ def apply_action(self, action_str: Optional[str]) -> ShortDeckPokerState: # stage of the game. new_state._increment_stage() new_state._reset_betting_round_state() - new_state._first_move_of_current_round = True + # new_state._first_move_of_current_round = True if not new_state.current_player.is_active: new_state._skip_counter += 1 assert not new_state.current_player.is_active @@ -385,31 +385,37 @@ def _increment_stage(self): # Progress from private cards to the flop. self._betting_stage = "flop" self._previous_betting_stage = "pre_flop" - if self._public_cards: - if len(self._public_cards) == 3: + if len(self._public_cards) >= 3: community_cards = self._public_cards[:3] self._poker_engine.table.community_cards += community_cards else: self._poker_engine.table.dealer.deal_flop(self._table) + self._public_information[ + self.betting_stage + ] = self._table.community_cards elif self._betting_stage == "flop": # Progress from flop to turn. self._betting_stage = "turn" self._previous_betting_stage = "flop" - if self._public_cards: - if len(self._public_cards) == 4: + if len(self._public_cards) >= 4: community_cards = self._public_cards[3:4] self._poker_engine.table.community_cards += community_cards else: self._poker_engine.table.dealer.deal_turn(self._table) + self._public_information[ + self.betting_stage + ] = self._table.community_cards elif self._betting_stage == "turn": # Progress from turn to river. self._betting_stage = "river" self._previous_betting_stage = "turn" - if self._public_cards: - if len(self._public_cards) == 5: + if len(self._public_cards) == 5: community_cards = self._public_cards[4:] self._poker_engine.table.community_cards += community_cards self._poker_engine.table.dealer.deal_river(self._table) + self._public_information[ + self.betting_stage + ] = self._table.community_cards elif self._betting_stage == "river": # Progress to the showdown. self._betting_stage = "show_down" @@ -437,9 +443,12 @@ def update_hole_cards_bayes(self): if "p_reach" in locals(): del p_reach action_sequence = [] + previous_betting_stage = "pre_flop" for idx, betting_stage in enumerate(self._history.keys()): - if self._first_move_of_current_round: - betting_stage = self._previous_betting_stage + if previous_betting_stage != betting_stage: + tmp = betting_stage + betting_stage = previous_betting_stage + previous_betting_stage = tmp n_actions_round = len(self._history[betting_stage]) for i in range(n_actions_round): action = self._history[betting_stage][i] @@ -504,12 +513,16 @@ def update_hole_cards_bayes(self): else: public_cards = self._public_information[betting_stage] public_cards_evals = [c.eval_card for c in public_cards] - infoset = self._info_set_helper( - starting_hand, - public_cards_evals, - action_sequence, - betting_stage, - ) + try: + infoset = self._info_set_helper( + starting_hand, + public_cards_evals, + action_sequence, + betting_stage, + ) + except: + import ipdb; + ipdb.set_trace() if self._offline_strategy[infoset].keys(): prob = self._offline_strategy[infoset][action] else: @@ -551,8 +564,8 @@ def deal_bayes(self): cards_selected += new_state._public_cards for card in cards_selected: new_state._table.dealer.deck.remove(card) - import ipdb; - ipdb.set_trace() + # import ipdb; + # ipdb.set_trace() return new_state # TODO add convenience method to supply public cards @@ -651,10 +664,10 @@ def n_players_started_round(self) -> bool: """Return n_players that started the round.""" return self._n_players_started_round - @property - def first_move_of_current_round(self) -> bool: - """Return boolfor first move of current round.""" - return self._first_move_of_current_round + # @property + # def first_move_of_current_round(self) -> bool: + # """Return boolfor first move of current round.""" + # return self._first_move_of_current_round @property def player_i(self) -> int: diff --git a/pluribus/poker/deck.py b/pluribus/poker/deck.py index 4901019d..a463058b 100644 --- a/pluribus/poker/deck.py +++ b/pluribus/poker/deck.py @@ -66,6 +66,8 @@ def remove(self, card): """Remove a specific card from the deck""" # TODO: is there any reason for them to be? # Maybe better to assert it's not - if card not in self._cards_in_deck: - self._cards_in_deck.remove(card) - self._dealt_cards.append(card) + c = card.eval_card + cards_in_deck = [x.eval_card for x in self._cards_in_deck] + if c in cards_in_deck: + index = cards_in_deck.index(c) + self._dealt_cards.append(self._cards_in_deck.pop(index)) diff --git a/research/test_methodology/RT.py b/research/test_methodology/RT.py index dbbaa804..ee5e0f1d 100644 --- a/research/test_methodology/RT.py +++ b/research/test_methodology/RT.py @@ -10,9 +10,7 @@ public_cards = [Card("ace", "spades"), Card("jack", "spades"), Card("queen", "hearts")] # we load a (trained) strategy agent1 = TrainedAgent("../blueprint_algo/results_2020_05_10_21_36_47_291425") - action_sequence = ["raise", "call", "raise"] - import ipdb - ipdb.set_trace() + action_sequence = ["raise", "call", "raise", "call", "call", "call"] agent_output = train(agent1.offline_strategy, public_cards, action_sequence, 100, 20, 20, 3) import ipdb; ipdb.set_trace() diff --git a/research/test_methodology/RT_cfr.py b/research/test_methodology/RT_cfr.py index 34346f55..de679571 100644 --- a/research/test_methodology/RT_cfr.py +++ b/research/test_methodology/RT_cfr.py @@ -76,7 +76,11 @@ def cfr(agent: Agent, state: ShortDeckPokerState, i: int, t: int) -> float: # cfr() elif ph == i: - I = state.info_set + try: + I = state.info_set + except: + import ipdb; + ipdb.set_trace() # calculate strategy logging.debug(f"About to Calculate Strategy, Regret: {agent.regret[I]}") logging.debug(f"Current regret: {agent.regret[I]}") @@ -107,7 +111,11 @@ def cfr(agent: Agent, state: ShortDeckPokerState, i: int, t: int) -> float: else: # import ipdb; # ipdb.set_trace() - Iph = state.info_set + try: + Iph = state.info_set + except: + import ipdb; + ipdb.set_trace() logging.debug(f"About to Calculate Strategy, Regret: {agent.regret[Iph]}") logging.debug(f"Current regret: {agent.regret[Iph]}") sigma = calculate_strategy(agent.regret, Iph, state) @@ -136,10 +144,10 @@ def get_game_state(state: ShortDeckPokerState, action_sequence: list): if not action_sequence: return state a = action_sequence.pop(0) - state.apply_action(a) + # state.apply_action(a) if a == "skip": a = action_sequence.pop(0) - state.apply_action(a) + # state.apply_action(a) new_state = state.apply_action(a) return get_game_state(new_state, action_sequence) @@ -184,8 +192,8 @@ def train( for i in range(n_players): # fixed position i # Create a new state. state: ShortDeckPokerState = current_game_state.deal_bayes() - import ipdb; - ipdb.set_trace() + # import ipdb; + # ipdb.set_trace() cfr(agent, state, i, t) if t < lcfr_threshold & t % discount_interval == 0: d = (t / discount_interval) / ((t / discount_interval) + 1) From a56ccca898253303df55197c604ab078f67a25b0 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Tue, 12 May 2020 18:39:43 -0400 Subject: [PATCH 10/42] fixing broken test, still a bug in deal_bayes method --- pluribus/games/short_deck/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pluribus/games/short_deck/state.py b/pluribus/games/short_deck/state.py index 87335ae9..23ac06d7 100644 --- a/pluribus/games/short_deck/state.py +++ b/pluribus/games/short_deck/state.py @@ -155,7 +155,7 @@ def __init__( load_pickle_files: bool = True, real_time_test: bool = False, offline_strategy: Dict = None, - public_cards: List[Card] = None + public_cards: List[Card] = [] ): """Initialise state.""" n_players = len(players) From ae6968dea931e5e8a330f788bb9c1c90b3f8a4a3 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Tue, 12 May 2020 22:31:57 -0400 Subject: [PATCH 11/42] it's doing something --- pluribus/games/short_deck/state.py | 45 +++++++++++++++++------------ research/test_methodology/RT.py | 2 +- research/test_methodology/RT_cfr.py | 4 --- 3 files changed, 27 insertions(+), 24 deletions(-) diff --git a/pluribus/games/short_deck/state.py b/pluribus/games/short_deck/state.py index 23ac06d7..ed4f433a 100644 --- a/pluribus/games/short_deck/state.py +++ b/pluribus/games/short_deck/state.py @@ -170,7 +170,7 @@ def __init__( self.info_set_lut = {} # Get a reference of the pot from the first player. self._table = PokerTable( - players=players, pot=players[0].pot, include_ranks=[10, 11, 12, 13, 14] + players=players, pot=players[0].pot, include_ranks=[12, 13, 14] #TODO, made the deck shorter, add 10, 11 back ) # Get a reference of the initial number of chips for the payout. self._initial_n_chips = players[0].n_chips @@ -442,15 +442,20 @@ def update_hole_cards_bayes(self): # TODO: is this bad? if "p_reach" in locals(): del p_reach - action_sequence = [] + action_sequence: Dict[str, List[str]] = collections.defaultdict(list) previous_betting_stage = "pre_flop" + first_action_round = False for idx, betting_stage in enumerate(self._history.keys()): - if previous_betting_stage != betting_stage: - tmp = betting_stage - betting_stage = previous_betting_stage - previous_betting_stage = tmp + # import ipdb; + # ipdb.set_trace() n_actions_round = len(self._history[betting_stage]) for i in range(n_actions_round): + # if i == 0: + # betting_stage = previous_betting_stage + # elif i == n_actions_round - 1: + # previous_betting_stage = betting_stage + # else: + # betting_stage = betting_round action = self._history[betting_stage][i] # TODO: maybe a method already exists for this? if betting_stage == "pre_flop": @@ -468,13 +473,10 @@ def update_hole_cards_bayes(self): c.eval_card for c in self._public_information[betting_stage] ] - if len( - set(starting_hand) - .union(set(opp_starting_hand)) - .union(set(public_evals)) - ) < (len(public_evals) + 4): + if len(set(opp_starting_hand).union(set(public_evals)).union(set(starting_hand))) < \ + len(opp_starting_hand) + len(starting_hand) + len(public_evals): prob = 0 - prob_reach_all_hands.append(prob) + num_hands += 1 else: num_hands += 1 @@ -491,15 +493,19 @@ def update_hole_cards_bayes(self): # check to see if the strategy exists, if not equal probability # TODO: is this hacky? problem with defaulting to 1 / 3, is that it # doesn't work for calculations that need to be made with the object's values + if self._offline_strategy[infoset].keys(): prob = self._offline_strategy[infoset][action] else: prob = 1 / len(self.legal_actions) - prob_reach_all_hands.append(prob) + prob_reach_all_hands.append(prob) + # import ipdb; + # ipdb.set_trace() + prob2 = sum(prob_reach_all_hands) / num_hands if "p_reach" not in locals(): - p_reach = sum(prob_reach_all_hands) / num_hands + p_reach = prob2 else: - p_reach *= p_reach + p_reach *= prob2 elif p_i == ph: public_evals = [ c.eval_card @@ -509,7 +515,6 @@ def update_hole_cards_bayes(self): len(public_evals) + 2 ): prob = 0 - prob_reach_all_hands.append(prob) else: public_cards = self._public_information[betting_stage] public_cards_evals = [c.eval_card for c in public_cards] @@ -527,11 +532,13 @@ def update_hole_cards_bayes(self): prob = self._offline_strategy[infoset][action] else: prob = 1 / len(self.legal_actions) + # import ipdb; + # ipdb.set_trace() if "p_reach" not in locals(): p_reach = prob else: - p_reach *= p_reach - action_sequence.append(action) + p_reach *= prob + action_sequence[betting_stage].append(action) self._starting_hand_probs[p_i][tuple(starting_hand)] = p_reach self._normalize_bayes() @@ -613,7 +620,7 @@ def _info_set_helper( "cards_cluster": cards_cluster, "history": [ {betting_stage: [str(action) for action in actions]} - for betting_stage, actions in self._history.items() + for betting_stage, actions in action_sequence.items() ], } return json.dumps( diff --git a/research/test_methodology/RT.py b/research/test_methodology/RT.py index ee5e0f1d..4157fccc 100644 --- a/research/test_methodology/RT.py +++ b/research/test_methodology/RT.py @@ -7,7 +7,7 @@ if __name__ == "__main__": utils.random.seed(38) - public_cards = [Card("ace", "spades"), Card("jack", "spades"), Card("queen", "hearts")] + public_cards = [Card("ace", "spades"), Card("queen", "spades"), Card("queen", "hearts")] # we load a (trained) strategy agent1 = TrainedAgent("../blueprint_algo/results_2020_05_10_21_36_47_291425") action_sequence = ["raise", "call", "raise", "call", "call", "call"] diff --git a/research/test_methodology/RT_cfr.py b/research/test_methodology/RT_cfr.py index de679571..d7c5ccfd 100644 --- a/research/test_methodology/RT_cfr.py +++ b/research/test_methodology/RT_cfr.py @@ -144,10 +144,8 @@ def get_game_state(state: ShortDeckPokerState, action_sequence: list): if not action_sequence: return state a = action_sequence.pop(0) - # state.apply_action(a) if a == "skip": a = action_sequence.pop(0) - # state.apply_action(a) new_state = state.apply_action(a) return get_game_state(new_state, action_sequence) @@ -192,8 +190,6 @@ def train( for i in range(n_players): # fixed position i # Create a new state. state: ShortDeckPokerState = current_game_state.deal_bayes() - # import ipdb; - # ipdb.set_trace() cfr(agent, state, i, t) if t < lcfr_threshold & t % discount_interval == 0: d = (t / discount_interval) / ((t / discount_interval) + 1) From 9fe1775297ecccd674ec1abf9fb959e55e44c2e8 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Tue, 12 May 2020 22:50:51 -0400 Subject: [PATCH 12/42] adding normal deck back and testing, seems like the regret is reasonable at least --- pluribus/games/short_deck/state.py | 2 +- research/test_methodology/RT.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pluribus/games/short_deck/state.py b/pluribus/games/short_deck/state.py index ed4f433a..da8a1466 100644 --- a/pluribus/games/short_deck/state.py +++ b/pluribus/games/short_deck/state.py @@ -170,7 +170,7 @@ def __init__( self.info_set_lut = {} # Get a reference of the pot from the first player. self._table = PokerTable( - players=players, pot=players[0].pot, include_ranks=[12, 13, 14] #TODO, made the deck shorter, add 10, 11 back + players=players, pot=players[0].pot, include_ranks=[10, 11, 12, 13, 14] ) # Get a reference of the initial number of chips for the payout. self._initial_n_chips = players[0].n_chips diff --git a/research/test_methodology/RT.py b/research/test_methodology/RT.py index 4157fccc..e7846f58 100644 --- a/research/test_methodology/RT.py +++ b/research/test_methodology/RT.py @@ -11,6 +11,6 @@ # we load a (trained) strategy agent1 = TrainedAgent("../blueprint_algo/results_2020_05_10_21_36_47_291425") action_sequence = ["raise", "call", "raise", "call", "call", "call"] - agent_output = train(agent1.offline_strategy, public_cards, action_sequence, 100, 20, 20, 3) + agent_output = train(agent1.offline_strategy, public_cards, action_sequence, 50, 20, 20, 3) import ipdb; ipdb.set_trace() From 09815cf20f4ac3b4c9d47dff0173d503693b5c91 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Wed, 13 May 2020 23:26:15 -0400 Subject: [PATCH 13/42] adding somewhat hacky update strategy --- research/test_methodology/RT.py | 2 +- research/test_methodology/RT_cfr.py | 56 +++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/research/test_methodology/RT.py b/research/test_methodology/RT.py index e7846f58..f61c23e2 100644 --- a/research/test_methodology/RT.py +++ b/research/test_methodology/RT.py @@ -11,6 +11,6 @@ # we load a (trained) strategy agent1 = TrainedAgent("../blueprint_algo/results_2020_05_10_21_36_47_291425") action_sequence = ["raise", "call", "raise", "call", "call", "call"] - agent_output = train(agent1.offline_strategy, public_cards, action_sequence, 50, 20, 20, 3) + agent_output = train(agent1.offline_strategy, public_cards, action_sequence, 150, 20, 20, 3, 1, 50) # TODO: back to 50 import ipdb; ipdb.set_trace() diff --git a/research/test_methodology/RT_cfr.py b/research/test_methodology/RT_cfr.py index d7c5ccfd..0fee6d3a 100644 --- a/research/test_methodology/RT_cfr.py +++ b/research/test_methodology/RT_cfr.py @@ -11,6 +11,57 @@ from pluribus.games.short_deck.state import * +def update_strategy(agent: Agent, state: ShortDeckPokerState, ph_test_node: int): + """ + + :param state: the game state + :param i: the player, i = 1 is always first to act and i = 2 is always second to act, but they take turns who + updates the strategy (only one strategy) + :return: nothing, updates action count in the strategy of actions chosen according to sigma, this simple choosing of + actions is what allows the algorithm to build up preference for one action over another in a given spot + """ + logging.debug("UPDATE STRATEGY") + logging.debug("########") + + logging.debug(f"P(h): {state.player_i}") + logging.debug(f"Betting Round {state._betting_stage}") + logging.debug(f"Community Cards {state._table.community_cards}") + logging.debug(f"Player 0 hole cards: {state.players[0].cards}") + logging.debug(f"Player 1 hole cards: {state.players[1].cards}") + logging.debug(f"Player 2 hole cards: {state.players[2].cards}") + logging.debug(f"Betting Action Correct?: {state.players}") + + ph = state.player_i # this is always the case no matter what i is + + if ph == ph_test_node: + try: + I = state.info_set + except: + import ipdb; + ipdb.set_trace() + # calculate regret + logging.debug(f"About to Calculate Strategy, Regret: {agent.regret[I]}") + logging.debug(f"Current regret: {agent.regret[I]}") + sigma = calculate_strategy(agent.regret, I, state) + logging.debug(f"Calculated Strategy for {I}: {sigma[I]}") + # choose an action based of sigma + try: + a = np.random.choice(list(sigma[I].keys()), 1, p=list(sigma[I].values()))[0] + logging.debug(f"ACTION SAMPLED: ph {state.player_i} ACTION: {a}") + except ValueError: + p = 1 / len(state.legal_actions) + probabilities = np.full(len(state.legal_actions), p) + a = np.random.choice(state.legal_actions, p=probabilities) + sigma[I] = {action: p for action in state.legal_actions} + logging.debug(f"ACTION SAMPLED: ph {state.player_i} ACTION: {a}") + # Increment the action counter. + agent.strategy[I][a] += 1 + logging.debug(f"Updated Strategy for {I}: {agent.strategy[I]}") + return + else: + return + + def calculate_strategy( regret: Dict[str, Dict[str, float]], I: str, state: ShortDeckPokerState, ): @@ -178,18 +229,23 @@ def train( lcfr_threshold: int, discount_interval: int, n_players: int, + update_interval: int, + update_threshold: int, ): """Train agent.""" utils.random.seed(38) agent = Agent() current_game_state = new_rt_game(3, offline_strategy, action_sequence, public_cards) + ph_test_node = current_game_state.player_i for t in trange(1, n_iterations + 1, desc="train iter"): if t == 2: logging.disable(logging.DEBUG) for i in range(n_players): # fixed position i # Create a new state. state: ShortDeckPokerState = current_game_state.deal_bayes() + if t % update_interval == 0 and t > update_threshold: + update_strategy(agent, state, ph_test_node) cfr(agent, state, i, t) if t < lcfr_threshold & t % discount_interval == 0: d = (t / discount_interval) / ((t / discount_interval) + 1) From 8750cebb415534ce2048796ec8dd040ea7c0cfee Mon Sep 17 00:00:00 2001 From: big-c-note Date: Fri, 15 May 2020 22:54:39 -0400 Subject: [PATCH 14/42] trying with better ph estimation --- pluribus/games/short_deck/state.py | 6 ++++++ research/test_methodology/RT.py | 17 ++++++++++++++--- research/test_methodology/RT_cfr.py | 3 ++- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/pluribus/games/short_deck/state.py b/pluribus/games/short_deck/state.py index da8a1466..70f90178 100644 --- a/pluribus/games/short_deck/state.py +++ b/pluribus/games/short_deck/state.py @@ -10,6 +10,7 @@ import joblib from itertools import combinations import random +import time import dill as pickle import numpy as np @@ -541,12 +542,17 @@ def update_hole_cards_bayes(self): action_sequence[betting_stage].append(action) self._starting_hand_probs[p_i][tuple(starting_hand)] = p_reach self._normalize_bayes() + # TODO: delete this? at least for our purposes we don't need it again + del self._offline_strategy def deal_bayes(self): + start = time.time() lut = self.info_set_lut self.info_set_lut = {} new_state = copy.deepcopy(self) new_state.info_set_lut = self.info_set_lut = lut + end = time.time() + print(f"Took {start - end} to load") players = list(range(len(self.players))) random.shuffle(players) diff --git a/research/test_methodology/RT.py b/research/test_methodology/RT.py index f61c23e2..b56ad33b 100644 --- a/research/test_methodology/RT.py +++ b/research/test_methodology/RT.py @@ -1,4 +1,8 @@ from typing import Dict +import joblib +import sys + +import dill as pickle from RT_cfr import * from pluribus.games.short_deck.state import * @@ -6,11 +10,18 @@ if __name__ == "__main__": - utils.random.seed(38) public_cards = [Card("ace", "spades"), Card("queen", "spades"), Card("queen", "hearts")] # we load a (trained) strategy - agent1 = TrainedAgent("../blueprint_algo/results_2020_05_10_21_36_47_291425") + #agent1 = TrainedAgent("../blueprint_algo/results_2020_05_10_21_36_47_291425") + # sorta hacky, but I loaded the average strategy above, now I'm replacing with + # the better strategy + offline_strategy = joblib.load('/Users/colin/Downloads/offline_strategy_285800.gz') + print(sys.getsizeof(offline_strategy)) + # agent1.offline_strategy = offline_strategy + # print(sys.getsizeof(agent1.offline_strategy)) action_sequence = ["raise", "call", "raise", "call", "call", "call"] - agent_output = train(agent1.offline_strategy, public_cards, action_sequence, 150, 20, 20, 3, 1, 50) # TODO: back to 50 + agent_output = train(offline_strategy, public_cards, action_sequence, 2400, 900, 900, 3, 2, 900) # TODO: back to 50 + with open('realtime-strategy-off-better-1500.pkl', 'wb') as file: + pickle.dump(agent_output, file) import ipdb; ipdb.set_trace() diff --git a/research/test_methodology/RT_cfr.py b/research/test_methodology/RT_cfr.py index 0fee6d3a..7f71e0f0 100644 --- a/research/test_methodology/RT_cfr.py +++ b/research/test_methodology/RT_cfr.py @@ -233,12 +233,13 @@ def train( update_threshold: int, ): """Train agent.""" - utils.random.seed(38) + utils.random.seed(36) agent = Agent() current_game_state = new_rt_game(3, offline_strategy, action_sequence, public_cards) ph_test_node = current_game_state.player_i for t in trange(1, n_iterations + 1, desc="train iter"): + print(t) if t == 2: logging.disable(logging.DEBUG) for i in range(n_players): # fixed position i From 1398134d66f9d95d614e045be6626ccca4c2e88c Mon Sep 17 00:00:00 2001 From: big-c-note Date: Sat, 16 May 2020 14:22:50 -0400 Subject: [PATCH 15/42] working out a few bugs --- pluribus/games/short_deck/state.py | 20 ++++++++++++-------- research/test_methodology/RT.py | 11 ++++++----- research/test_methodology/RT_cfr.py | 3 ++- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/pluribus/games/short_deck/state.py b/pluribus/games/short_deck/state.py index 70f90178..7cd75bc9 100644 --- a/pluribus/games/short_deck/state.py +++ b/pluribus/games/short_deck/state.py @@ -393,7 +393,7 @@ def _increment_stage(self): self._poker_engine.table.dealer.deal_flop(self._table) self._public_information[ self.betting_stage - ] = self._table.community_cards + ] = self._table.community_cards.copy() elif self._betting_stage == "flop": # Progress from flop to turn. self._betting_stage = "turn" @@ -405,7 +405,7 @@ def _increment_stage(self): self._poker_engine.table.dealer.deal_turn(self._table) self._public_information[ self.betting_stage - ] = self._table.community_cards + ] = self._table.community_cards.copy() elif self._betting_stage == "turn": # Progress from turn to river. self._betting_stage = "river" @@ -413,10 +413,11 @@ def _increment_stage(self): if len(self._public_cards) == 5: community_cards = self._public_cards[4:] self._poker_engine.table.community_cards += community_cards - self._poker_engine.table.dealer.deal_river(self._table) + else: + self._poker_engine.table.dealer.deal_river(self._table) self._public_information[ self.betting_stage - ] = self._table.community_cards + ] = self._table.community_cards.copy() elif self._betting_stage == "river": # Progress to the showdown. self._betting_stage = "show_down" @@ -458,6 +459,9 @@ def update_hole_cards_bayes(self): # else: # betting_stage = betting_round action = self._history[betting_stage][i] + while action == 'skip': + i += 1 # should be ok, action sequences don't end in skip + action = self._history[betting_stage][i] # TODO: maybe a method already exists for this? if betting_stage == "pre_flop": ph = (i + 2) % n_players @@ -495,9 +499,9 @@ def update_hole_cards_bayes(self): # TODO: is this hacky? problem with defaulting to 1 / 3, is that it # doesn't work for calculations that need to be made with the object's values - if self._offline_strategy[infoset].keys(): + try: # TODO: with or without keys prob = self._offline_strategy[infoset][action] - else: + except KeyError: prob = 1 / len(self.legal_actions) prob_reach_all_hands.append(prob) # import ipdb; @@ -529,9 +533,9 @@ def update_hole_cards_bayes(self): except: import ipdb; ipdb.set_trace() - if self._offline_strategy[infoset].keys(): + try: prob = self._offline_strategy[infoset][action] - else: + except KeyError: prob = 1 / len(self.legal_actions) # import ipdb; # ipdb.set_trace() diff --git a/research/test_methodology/RT.py b/research/test_methodology/RT.py index b56ad33b..f9dfa3c6 100644 --- a/research/test_methodology/RT.py +++ b/research/test_methodology/RT.py @@ -10,17 +10,18 @@ if __name__ == "__main__": - public_cards = [Card("ace", "spades"), Card("queen", "spades"), Card("queen", "hearts")] + #public_cards = [Card("ace", "spades"), Card("queen", "spades"), Card("queen", "hearts")] + public_cards = [] # we load a (trained) strategy - #agent1 = TrainedAgent("../blueprint_algo/results_2020_05_10_21_36_47_291425") + agent1 = TrainedAgent("../blueprint_algo/results_2020_05_10_21_36_47_291425") # sorta hacky, but I loaded the average strategy above, now I'm replacing with # the better strategy offline_strategy = joblib.load('/Users/colin/Downloads/offline_strategy_285800.gz') - print(sys.getsizeof(offline_strategy)) + #print(sys.getsizeof(offline_strategy)) # agent1.offline_strategy = offline_strategy # print(sys.getsizeof(agent1.offline_strategy)) - action_sequence = ["raise", "call", "raise", "call", "call", "call"] - agent_output = train(offline_strategy, public_cards, action_sequence, 2400, 900, 900, 3, 2, 900) # TODO: back to 50 + action_sequence = ["raise", "call", "call", "call", "call", "call", "call", "call", "call", "call", "call"] + agent_output = train(offline_strategy, public_cards, action_sequence, 200, 60, 60, 3, 2, 60) # TODO: back to 50 with open('realtime-strategy-off-better-1500.pkl', 'wb') as file: pickle.dump(agent_output, file) import ipdb; diff --git a/research/test_methodology/RT_cfr.py b/research/test_methodology/RT_cfr.py index 7f71e0f0..38a8f23c 100644 --- a/research/test_methodology/RT_cfr.py +++ b/research/test_methodology/RT_cfr.py @@ -197,13 +197,14 @@ def get_game_state(state: ShortDeckPokerState, action_sequence: list): a = action_sequence.pop(0) if a == "skip": a = action_sequence.pop(0) + print(a) new_state = state.apply_action(a) return get_game_state(new_state, action_sequence) # added some flags for RT def new_rt_game( - n_players: int, offline_strategy: Dict, action_sequence, public_cards, real_time_test=True + n_players: int, offline_strategy: Dict, action_sequence, public_cards=[], real_time_test=True ) -> ShortDeckPokerState: """Create a new game of short deck poker.""" pot = Pot() From e341eadc7d6e365e613d2ef716ebeffc11b416ad Mon Sep 17 00:00:00 2001 From: big-c-note Date: Sun, 17 May 2020 21:31:11 -0400 Subject: [PATCH 16/42] rebasing, confirming RT is running before refactor --- research/test_methodology/RT.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/research/test_methodology/RT.py b/research/test_methodology/RT.py index f9dfa3c6..66f02c90 100644 --- a/research/test_methodology/RT.py +++ b/research/test_methodology/RT.py @@ -16,12 +16,12 @@ agent1 = TrainedAgent("../blueprint_algo/results_2020_05_10_21_36_47_291425") # sorta hacky, but I loaded the average strategy above, now I'm replacing with # the better strategy - offline_strategy = joblib.load('/Users/colin/Downloads/offline_strategy_285800.gz') + #offline_strategy = joblib.load('/Users/colin/Downloads/offline_strategy_285800.gz') #print(sys.getsizeof(offline_strategy)) # agent1.offline_strategy = offline_strategy # print(sys.getsizeof(agent1.offline_strategy)) - action_sequence = ["raise", "call", "call", "call", "call", "call", "call", "call", "call", "call", "call"] - agent_output = train(offline_strategy, public_cards, action_sequence, 200, 60, 60, 3, 2, 60) # TODO: back to 50 + action_sequence = ["raise", "call", "call", "call","call"] + agent_output = train(agent1.offline_strategy, public_cards, action_sequence, 200, 60, 60, 3, 2, 60) # TODO: back to 50 with open('realtime-strategy-off-better-1500.pkl', 'wb') as file: pickle.dump(agent_output, file) import ipdb; From 682298d7d6b85461fa75185198ae7b3c6e50a940 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Sun, 17 May 2020 23:12:31 -0400 Subject: [PATCH 17/42] moving get_game_state to ShortDeckPokerState class --- pluribus/games/short_deck/state.py | 14 +++++++++++++- research/test_methodology/RT_cfr.py | 7 ++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/pluribus/games/short_deck/state.py b/pluribus/games/short_deck/state.py index 7cd75bc9..bc7898f9 100644 --- a/pluribus/games/short_deck/state.py +++ b/pluribus/games/short_deck/state.py @@ -584,9 +584,21 @@ def deal_bayes(self): # import ipdb; # ipdb.set_trace() return new_state - # TODO add convenience method to supply public cards + def get_game_state(self, action_sequence: list): + """ + Follow through the action sequence provided to get current node. + :param action_sequence: List of actions without 'skip' + """ + if not action_sequence: + return self + a = action_sequence.pop(0) + if a == "skip": + a = action_sequence.pop(0) + new_state = self.apply_action(a) + return new_state.get_game_state(action_sequence) + def _get_starting_hand(self, player_idx: int): """Get starting hand based on probability of reach""" starting_hand_idxs = list(range(len(self._starting_hand_probs[player_idx].keys()))) diff --git a/research/test_methodology/RT_cfr.py b/research/test_methodology/RT_cfr.py index 38a8f23c..22120764 100644 --- a/research/test_methodology/RT_cfr.py +++ b/research/test_methodology/RT_cfr.py @@ -149,7 +149,7 @@ def cfr(agent: Agent, state: ShortDeckPokerState, i: int, t: int) -> float: logging.debug(f"Got EV for {a}: {voa[a]}") vo += sigma[I][a] * voa[a] logging.debug( - f"""Added to Node EV for ACTION: {a} INFOSET: {I} + f"""Added to Node EV for ACTION: {a} INFOSET: {I} STRATEGY: {sigma[I][a]}: {sigma[I][a] * voa[a]}""" ) logging.debug(f"Updated EV at {I}: {vo}") @@ -189,8 +189,6 @@ def cfr(agent: Agent, state: ShortDeckPokerState, i: int, t: int) -> float: return cfr(agent, new_state, i, t) -# this is a decent function for loading up a particular action sequence -def get_game_state(state: ShortDeckPokerState, action_sequence: list): """Follow through the action sequence provided to get current node""" if not action_sequence: return state @@ -201,7 +199,6 @@ def get_game_state(state: ShortDeckPokerState, action_sequence: list): new_state = state.apply_action(a) return get_game_state(new_state, action_sequence) - # added some flags for RT def new_rt_game( n_players: int, offline_strategy: Dict, action_sequence, public_cards=[], real_time_test=True @@ -215,7 +212,7 @@ def new_rt_game( state = ShortDeckPokerState( players=players, offline_strategy=offline_strategy, real_time_test=real_time_test, public_cards=public_cards ) - current_game_state = get_game_state(state, action_sequence) + current_game_state = state.get_game_state(action_sequence) # decided to make this a one time method rather than something that always updates # reason being: we won't need it except for a few choice nodes current_game_state.update_hole_cards_bayes() From bf3cfadc7ad587aa9c11f32620fa9406bf55cc6d Mon Sep 17 00:00:00 2001 From: big-c-note Date: Sun, 17 May 2020 23:18:17 -0400 Subject: [PATCH 18/42] removing leftover code --- research/test_methodology/RT_cfr.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/research/test_methodology/RT_cfr.py b/research/test_methodology/RT_cfr.py index 22120764..9342516c 100644 --- a/research/test_methodology/RT_cfr.py +++ b/research/test_methodology/RT_cfr.py @@ -188,17 +188,6 @@ def cfr(agent: Agent, state: ShortDeckPokerState, i: int, t: int) -> float: new_state: ShortDeckPokerState = state.apply_action(a) return cfr(agent, new_state, i, t) - - """Follow through the action sequence provided to get current node""" - if not action_sequence: - return state - a = action_sequence.pop(0) - if a == "skip": - a = action_sequence.pop(0) - print(a) - new_state = state.apply_action(a) - return get_game_state(new_state, action_sequence) - # added some flags for RT def new_rt_game( n_players: int, offline_strategy: Dict, action_sequence, public_cards=[], real_time_test=True From ebf0a9e7cf70af29c28efadb9e83a86ae1c018d1 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Mon, 18 May 2020 14:56:43 -0400 Subject: [PATCH 19/42] moving agent class to its own file --- pluribus/games/short_deck/agent.py | 99 +++++++++++++++++++++++++++++ research/test_methodology/RT.py | 22 +++---- research/test_methodology/RT_cfr.py | 2 +- 3 files changed, 111 insertions(+), 12 deletions(-) create mode 100644 pluribus/games/short_deck/agent.py diff --git a/pluribus/games/short_deck/agent.py b/pluribus/games/short_deck/agent.py new file mode 100644 index 00000000..ce924831 --- /dev/null +++ b/pluribus/games/short_deck/agent.py @@ -0,0 +1,99 @@ +import os +import collections +from typing import Dict +import joblib + + +class Agent: + # TODO(fedden): Note from the supplementary material, the data here will + # need to be lower precision: "To save memory, regrets were + # stored using 4-byte integers rather than 8-byte doubles. + # There was also a floor on regret at -310,000,000 for every + # action. This made it easier to unprune actions that were + # initially pruned but later improved. This also prevented + # integer overflows". + def __init__(self): + self.strategy = collections.defaultdict( + lambda: collections.defaultdict(lambda: 0) + ) + self.regret = collections.defaultdict( + lambda: collections.defaultdict(lambda: 0) + ) + + +# TODO: Change to Leon's newest iteration on this method +class TrainedAgent(Agent): + """ + Agent who has been trained + Points to a folder whose strategies is calculated from regret and then averaged + """ + + def __init__(self, directory: str): + super().__init__() + self.offline_strategy = self._load_regret(directory) + + # TODO: the following could use a refactor, just getting through this + # rather quickly + def _calculate_strategy( + self, + regret: Dict[str, Dict[str, float]], + sigma: Dict[int, Dict[str, Dict[str, float]]], + I: str, + ): + """ + Get strategy from regret + """ + rsum = sum([max(x, 0) for x in regret[I].values()]) + ACTIONS = regret[I].keys() # TODO: this is hacky, might be a better way + for a in ACTIONS: + if rsum > 0: + sigma[I][a] = max(regret[I][a], 0) / rsum + else: + sigma[I][a] = 1 / len(ACTIONS) + return sigma + + def _average_strategy(self, directory: str): + files = [ + x + for x in os.listdir(directory) + if os.path.isfile(os.path.join(directory, x)) + ] + + offline_strategy: Dict = collections.defaultdict( + lambda: collections.defaultdict(lambda: 0) + ) + strategy_tmp = collections.defaultdict( + lambda: collections.defaultdict(lambda: 0) + ) + + for idx, f in enumerate(files): + if f in ["config.yaml", "strategy.gz"]: + continue + + regret_dict = joblib.load(directory + "/" + f)["regret"] + sigma = collections.defaultdict( + lambda: collections.defaultdict(lambda: 1 / 3) + ) + + for info_set, regret in sorted(regret_dict.items()): + sigma = self._calculate_strategy(regret_dict, sigma, info_set) + + for info_set, strategy in sigma.items(): + for action, probability in strategy.items(): + try: + strategy_tmp[info_set][action] += probability + except KeyError: + strategy_tmp[info_set][action] = probability + + for info_set, strategy in sorted(strategy_tmp.items()): + norm = sum(list(strategy.values())) + for action, probability in strategy.items(): + try: + offline_strategy[info_set][action] += probability / norm + except KeyError: + offline_strategy[info_set][action] = probability / norm + + return offline_strategy + + def _load_regret(self, directory: str): + return self._average_strategy(directory) diff --git a/research/test_methodology/RT.py b/research/test_methodology/RT.py index 66f02c90..2c7d2ac3 100644 --- a/research/test_methodology/RT.py +++ b/research/test_methodology/RT.py @@ -1,28 +1,28 @@ -from typing import Dict -import joblib -import sys - import dill as pickle from RT_cfr import * from pluribus.games.short_deck.state import * +from pluribus.games.short_deck.agent import * from pluribus.poker.card import Card if __name__ == "__main__": - #public_cards = [Card("ace", "spades"), Card("queen", "spades"), Card("queen", "hearts")] + # public_cards = [Card("ace", "spades"), Card("queen", "spades"), Card("queen", "hearts")] public_cards = [] # we load a (trained) strategy agent1 = TrainedAgent("../blueprint_algo/results_2020_05_10_21_36_47_291425") # sorta hacky, but I loaded the average strategy above, now I'm replacing with # the better strategy - #offline_strategy = joblib.load('/Users/colin/Downloads/offline_strategy_285800.gz') - #print(sys.getsizeof(offline_strategy)) + # offline_strategy = joblib.load('/Users/colin/Downloads/offline_strategy_285800.gz') + # print(sys.getsizeof(offline_strategy)) # agent1.offline_strategy = offline_strategy # print(sys.getsizeof(agent1.offline_strategy)) - action_sequence = ["raise", "call", "call", "call","call"] - agent_output = train(agent1.offline_strategy, public_cards, action_sequence, 200, 60, 60, 3, 2, 60) # TODO: back to 50 - with open('realtime-strategy-off-better-1500.pkl', 'wb') as file: + action_sequence = ["raise", "call", "call", "call", "call"] + agent_output = train( + agent1.offline_strategy, public_cards, action_sequence, 40, 6, 6, 3, 2, 6 + ) # TODO: back to 50 + with open("realtime-strategy-moved-agent.pkl", "wb") as file: pickle.dump(agent_output, file) - import ipdb; + import ipdb + ipdb.set_trace() diff --git a/research/test_methodology/RT_cfr.py b/research/test_methodology/RT_cfr.py index 9342516c..51802d6b 100644 --- a/research/test_methodology/RT_cfr.py +++ b/research/test_methodology/RT_cfr.py @@ -9,7 +9,7 @@ from tqdm import trange from pluribus.games.short_deck.state import * - +from pluribus.games.short_deck.agent import * def update_strategy(agent: Agent, state: ShortDeckPokerState, ph_test_node: int): """ From da97637b9e23c6d38d8b152b2f6d0c668f42a03c Mon Sep 17 00:00:00 2001 From: big-c-note Date: Mon, 18 May 2020 16:23:36 -0400 Subject: [PATCH 20/42] forgot to add in last commit, removing agent class --- pluribus/games/short_deck/state.py | 145 ++++------------------------- 1 file changed, 20 insertions(+), 125 deletions(-) diff --git a/pluribus/games/short_deck/state.py b/pluribus/games/short_deck/state.py index bc7898f9..dc136e6e 100644 --- a/pluribus/games/short_deck/state.py +++ b/pluribus/games/short_deck/state.py @@ -7,7 +7,6 @@ import operator import os from typing import Any, Dict, List, Optional, Tuple -import joblib from itertools import combinations import random import time @@ -37,7 +36,8 @@ def new_game( ] if info_set_lut: # Don't reload massive files, it takes ages. - state = ShortDeckPokerState(players=players, load_pickle_files=False, **kwargs) + state = ShortDeckPokerState(players=players, + load_pickle_files=False, **kwargs) state.info_set_lut = info_set_lut else: # Load massive files. @@ -45,101 +45,6 @@ def new_game( return state -# TODO: there is probably a better place for agent related items -class Agent: - # TODO(fedden): Note from the supplementary material, the data here will - # need to be lower precision: "To save memory, regrets were - # stored using 4-byte integers rather than 8-byte doubles. - # There was also a floor on regret at -310,000,000 for every - # action. This made it easier to unprune actions that were - # initially pruned but later improved. This also prevented - # integer overflows". - def __init__(self): - self.strategy = collections.defaultdict( - lambda: collections.defaultdict(lambda: 0) - ) - self.regret = collections.defaultdict( - lambda: collections.defaultdict(lambda: 0) - ) - - -# TODO: Change to Leon's newest iteration on this method -class TrainedAgent(Agent): - """ - Agent who has been trained - Points to a folder whose strategies is calculated from regret and then averaged - """ - - def __init__(self, directory: str): - super().__init__() - self.offline_strategy = self._load_regret(directory) - - # TODO: the following could use a refactor, just getting through this rather quickly - def _calculate_strategy( - self, - regret: Dict[str, Dict[str, float]], - sigma: Dict[int, Dict[str, Dict[str, float]]], - I: str, - ): - """ - Get strategy from regret - """ - rsum = sum([max(x, 0) for x in regret[I].values()]) - ACTIONS = regret[I].keys() # TODO: this is hacky, might be a better way - for a in ACTIONS: - if rsum > 0: - sigma[I][a] = max(regret[I][a], 0) / rsum - else: - sigma[I][a] = 1 / len(ACTIONS) - return sigma - - def _average_strategy(self, directory: str): - files = [ - x - for x in os.listdir(directory) - if os.path.isfile(os.path.join(directory, x)) - ] - - offline_strategy = collections.defaultdict( - lambda: collections.defaultdict(lambda: 0) - ) - strategy_tmp = collections.defaultdict( - lambda: collections.defaultdict(lambda: 0) - ) - - for idx, f in enumerate(files): - if f in ["config.yaml", "strategy.gz"]: - continue - - regret_dict = joblib.load(directory + "/" + f)["regret"] - sigma = collections.defaultdict( - lambda: collections.defaultdict(lambda: 1 / 3) - ) - - for info_set, regret in sorted(regret_dict.items()): - sigma = self._calculate_strategy(regret_dict, sigma, info_set) - - for info_set, strategy in sigma.items(): - for action, probability in strategy.items(): - try: - strategy_tmp[info_set][action] += probability - except KeyError: - strategy_tmp[info_set][action] = probability - - for info_set, strategy in sorted(strategy_tmp.items()): - norm = sum(list(strategy.values())) - for action, probability in strategy.items(): - try: - offline_strategy[info_set][action] += probability / norm - except KeyError: - offline_strategy[info_set][action] = probability / norm - - return offline_strategy - - def _load_regret(self, directory: str): - return self._average_strategy(directory) - - class ShortDeckPokerState: """The state of a Short Deck Poker game at some given point in time. @@ -219,7 +124,8 @@ def __init__( for player in self.players: player.is_turn = False self.current_player.is_turn = True - # TODO add attribute of public_cards, that can be supplied by convenience method + # TODO add attribute of public_cards, that can be supplied by + # convenience method self._public_cards = public_cards if public_cards: assert len(public_cards) in {3, 4, 5} @@ -329,11 +235,7 @@ def apply_action(self, action_str: Optional[str]) -> ShortDeckPokerState: # Now check if the game is terminal. if new_state._betting_stage in {"terminal", "show_down"}: # Distribute winnings. - try: - new_state._poker_engine.compute_winners() - except: - import ipdb; - ipdb.set_trace() + new_state._poker_engine.compute_winners() break for player in new_state.players: player.is_turn = False @@ -387,8 +289,8 @@ def _increment_stage(self): self._betting_stage = "flop" self._previous_betting_stage = "pre_flop" if len(self._public_cards) >= 3: - community_cards = self._public_cards[:3] - self._poker_engine.table.community_cards += community_cards + community_cards = self._public_cards[:3] + self._poker_engine.table.community_cards += community_cards else: self._poker_engine.table.dealer.deal_flop(self._table) self._public_information[ @@ -399,8 +301,8 @@ def _increment_stage(self): self._betting_stage = "turn" self._previous_betting_stage = "flop" if len(self._public_cards) >= 4: - community_cards = self._public_cards[3:4] - self._poker_engine.table.community_cards += community_cards + community_cards = self._public_cards[3:4] + self._poker_engine.table.community_cards += community_cards else: self._poker_engine.table.dealer.deal_turn(self._table) self._public_information[ @@ -411,8 +313,8 @@ def _increment_stage(self): self._betting_stage = "river" self._previous_betting_stage = "turn" if len(self._public_cards) == 5: - community_cards = self._public_cards[4:] - self._poker_engine.table.community_cards += community_cards + community_cards = self._public_cards[4:] + self._poker_engine.table.community_cards += community_cards else: self._poker_engine.table.dealer.deal_river(self._table) self._public_information[ @@ -460,7 +362,7 @@ def update_hole_cards_bayes(self): # betting_stage = betting_round action = self._history[betting_stage][i] while action == 'skip': - i += 1 # should be ok, action sequences don't end in skip + i += 1 # action sequences don't end in skip action = self._history[betting_stage][i] # TODO: maybe a method already exists for this? if betting_stage == "pre_flop": @@ -499,7 +401,7 @@ def update_hole_cards_bayes(self): # TODO: is this hacky? problem with defaulting to 1 / 3, is that it # doesn't work for calculations that need to be made with the object's values - try: # TODO: with or without keys + try: # TODO: with or without keys prob = self._offline_strategy[infoset][action] except KeyError: prob = 1 / len(self.legal_actions) @@ -523,22 +425,17 @@ def update_hole_cards_bayes(self): else: public_cards = self._public_information[betting_stage] public_cards_evals = [c.eval_card for c in public_cards] - try: - infoset = self._info_set_helper( - starting_hand, - public_cards_evals, - action_sequence, - betting_stage, - ) - except: - import ipdb; - ipdb.set_trace() + infoset = self._info_set_helper( + starting_hand, + public_cards_evals, + action_sequence, + betting_stage, + ) + # TODO: Check this try: prob = self._offline_strategy[infoset][action] except KeyError: prob = 1 / len(self.legal_actions) - # import ipdb; - # ipdb.set_trace() if "p_reach" not in locals(): p_reach = prob else: @@ -581,8 +478,6 @@ def deal_bayes(self): cards_selected += new_state._public_cards for card in cards_selected: new_state._table.dealer.deck.remove(card) - # import ipdb; - # ipdb.set_trace() return new_state # TODO add convenience method to supply public cards From a982fffe7a0425bfe54d63f5b12a95d4d1206fb3 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Mon, 18 May 2020 18:07:12 -0400 Subject: [PATCH 21/42] removing agent strategy from the state class --- pluribus/games/short_deck/state.py | 22 +++++++++------- research/test_methodology/RT.py | 2 +- research/test_methodology/RT_cfr.py | 41 +++++++++++------------------ 3 files changed, 29 insertions(+), 36 deletions(-) diff --git a/pluribus/games/short_deck/state.py b/pluribus/games/short_deck/state.py index dc136e6e..04834d74 100644 --- a/pluribus/games/short_deck/state.py +++ b/pluribus/games/short_deck/state.py @@ -60,7 +60,6 @@ def __init__( pickle_dir: str = ".", load_pickle_files: bool = True, real_time_test: bool = False, - offline_strategy: Dict = None, public_cards: List[Card] = [] ): """Initialise state.""" @@ -133,8 +132,6 @@ def __init__( # only want to do these actions in real game play, as they are slow if self.real_time_test: # must have offline strategy loaded up - assert offline_strategy - self._offline_strategy = offline_strategy self._starting_hand_probs = self._initialize_starting_hands() # TODO: We might not need this cards_in_deck = self._table.dealer.deck._cards_in_deck @@ -337,7 +334,7 @@ def _normalize_bayes(self): for starting_hand, prob in self._starting_hand_probs[player].items(): self._starting_hand_probs[player][starting_hand] = prob / total_prob - def update_hole_cards_bayes(self): + def _update_hole_cards_bayes(self, offline_strategy: Dict): """Get probability of reach for each pair of hole cards for each player""" n_players = len(self._table.players) player_indices: List[int] = [p_i for p_i in range(n_players)] @@ -402,7 +399,7 @@ def update_hole_cards_bayes(self): # doesn't work for calculations that need to be made with the object's values try: # TODO: with or without keys - prob = self._offline_strategy[infoset][action] + prob = offline_strategy[infoset][action] except KeyError: prob = 1 / len(self.legal_actions) prob_reach_all_hands.append(prob) @@ -433,7 +430,7 @@ def update_hole_cards_bayes(self): ) # TODO: Check this try: - prob = self._offline_strategy[infoset][action] + prob = offline_strategy[infoset][action] except KeyError: prob = 1 / len(self.legal_actions) if "p_reach" not in locals(): @@ -444,7 +441,6 @@ def update_hole_cards_bayes(self): self._starting_hand_probs[p_i][tuple(starting_hand)] = p_reach self._normalize_bayes() # TODO: delete this? at least for our purposes we don't need it again - del self._offline_strategy def deal_bayes(self): start = time.time() @@ -481,18 +477,24 @@ def deal_bayes(self): return new_state # TODO add convenience method to supply public cards - def get_game_state(self, action_sequence: list): + def load_game_state(self, offline_strategy: Dict, action_sequence: list): """ Follow through the action sequence provided to get current node. :param action_sequence: List of actions without 'skip' """ if not action_sequence: - return self + # TODO: not 100 percent sure I need to deep copy + lut = self.info_set_lut + self.info_set_lut = {} + new_state = copy.deepcopy(self) + new_state.info_set_lut = self.info_set_lut = lut + new_state._update_hole_cards_bayes(offline_strategy) + return new_state a = action_sequence.pop(0) if a == "skip": a = action_sequence.pop(0) new_state = self.apply_action(a) - return new_state.get_game_state(action_sequence) + return new_state.load_game_state(offline_strategy, action_sequence) def _get_starting_hand(self, player_idx: int): """Get starting hand based on probability of reach""" diff --git a/research/test_methodology/RT.py b/research/test_methodology/RT.py index 2c7d2ac3..caf557e4 100644 --- a/research/test_methodology/RT.py +++ b/research/test_methodology/RT.py @@ -21,7 +21,7 @@ agent_output = train( agent1.offline_strategy, public_cards, action_sequence, 40, 6, 6, 3, 2, 6 ) # TODO: back to 50 - with open("realtime-strategy-moved-agent.pkl", "wb") as file: + with open("realtime-strategy-refactor-game-state.pkl", "wb") as file: pickle.dump(agent_output, file) import ipdb diff --git a/research/test_methodology/RT_cfr.py b/research/test_methodology/RT_cfr.py index 51802d6b..72fad794 100644 --- a/research/test_methodology/RT_cfr.py +++ b/research/test_methodology/RT_cfr.py @@ -8,9 +8,11 @@ from tqdm import trange +from pluribus import utils from pluribus.games.short_deck.state import * from pluribus.games.short_deck.agent import * + def update_strategy(agent: Agent, state: ShortDeckPokerState, ph_test_node: int): """ @@ -37,7 +39,8 @@ def update_strategy(agent: Agent, state: ShortDeckPokerState, ph_test_node: int) try: I = state.info_set except: - import ipdb; + import ipdb + ipdb.set_trace() # calculate regret logging.debug(f"About to Calculate Strategy, Regret: {agent.regret[I]}") @@ -63,7 +66,7 @@ def update_strategy(agent: Agent, state: ShortDeckPokerState, ph_test_node: int) def calculate_strategy( - regret: Dict[str, Dict[str, float]], I: str, state: ShortDeckPokerState, + regret: Dict[str, Dict[str, float]], I: str, state: ShortDeckPokerState, ): """ @@ -130,7 +133,8 @@ def cfr(agent: Agent, state: ShortDeckPokerState, i: int, t: int) -> float: try: I = state.info_set except: - import ipdb; + import ipdb + ipdb.set_trace() # calculate strategy logging.debug(f"About to Calculate Strategy, Regret: {agent.regret[I]}") @@ -165,7 +169,8 @@ def cfr(agent: Agent, state: ShortDeckPokerState, i: int, t: int) -> float: try: Iph = state.info_set except: - import ipdb; + import ipdb + ipdb.set_trace() logging.debug(f"About to Calculate Strategy, Regret: {agent.regret[Iph]}") logging.debug(f"Current regret: {agent.regret[Iph]}") @@ -188,29 +193,10 @@ def cfr(agent: Agent, state: ShortDeckPokerState, i: int, t: int) -> float: new_state: ShortDeckPokerState = state.apply_action(a) return cfr(agent, new_state, i, t) -# added some flags for RT -def new_rt_game( - n_players: int, offline_strategy: Dict, action_sequence, public_cards=[], real_time_test=True -) -> ShortDeckPokerState: - """Create a new game of short deck poker.""" - pot = Pot() - players = [ - ShortDeckPokerPlayer(player_i=player_i, initial_chips=10000, pot=pot) - for player_i in range(n_players) - ] - state = ShortDeckPokerState( - players=players, offline_strategy=offline_strategy, real_time_test=real_time_test, public_cards=public_cards - ) - current_game_state = state.get_game_state(action_sequence) - # decided to make this a one time method rather than something that always updates - # reason being: we won't need it except for a few choice nodes - current_game_state.update_hole_cards_bayes() - return current_game_state - def train( offline_strategy: Dict, - public_cards, + public_cards: list, action_sequence: list, n_iterations: int, lcfr_threshold: int, @@ -223,7 +209,12 @@ def train( utils.random.seed(36) agent = Agent() - current_game_state = new_rt_game(3, offline_strategy, action_sequence, public_cards) + state: ShortDeckPokerState = new_game(3, real_time_test=True, public_cards=public_cards) + current_game_state: ShortDeckPokerState = state.load_game_state( + offline_strategy, + action_sequence + ) + del offline_strategy ph_test_node = current_game_state.player_i for t in trange(1, n_iterations + 1, desc="train iter"): print(t) From cd48a7364e0f1b5258d4ae2a04214384648db853 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Tue, 19 May 2020 01:12:43 -0400 Subject: [PATCH 22/42] refactoring useages of the deck class and card class --- pluribus/games/short_deck/state.py | 206 +++++++++++++++-------------- pluribus/poker/card.py | 4 +- pluribus/poker/deck.py | 10 +- research/test_methodology/RT.py | 3 +- 4 files changed, 112 insertions(+), 111 deletions(-) diff --git a/pluribus/games/short_deck/state.py b/pluribus/games/short_deck/state.py index 04834d74..4a763d5c 100644 --- a/pluribus/games/short_deck/state.py +++ b/pluribus/games/short_deck/state.py @@ -123,9 +123,6 @@ def __init__( for player in self.players: player.is_turn = False self.current_player.is_turn = True - # TODO add attribute of public_cards, that can be supplied by - # convenience method - self._public_cards = public_cards if public_cards: assert len(public_cards) in {3, 4, 5} self._public_cards = public_cards @@ -133,10 +130,6 @@ def __init__( if self.real_time_test: # must have offline strategy loaded up self._starting_hand_probs = self._initialize_starting_hands() - # TODO: We might not need this - cards_in_deck = self._table.dealer.deck._cards_in_deck - self._evals = [c.eval_card for c in cards_in_deck] - self._evals_to_cards = {i.eval_card: i for i in cards_in_deck} def __repr__(self): """Return a helpful description of object in strings and debugger.""" @@ -329,34 +322,30 @@ def _increment_stage(self): def _normalize_bayes(self): """Normalize probability of reach for each player""" n_players = len(self.players) - for player in range(n_players): - total_prob = sum(self._starting_hand_probs[player].values()) - for starting_hand, prob in self._starting_hand_probs[player].items(): - self._starting_hand_probs[player][starting_hand] = prob / total_prob + for p_i in range(n_players): + total_prob = sum(self._starting_hand_probs[p_i].values()) + for starting_hand, prob in self._starting_hand_probs[p_i].items(): + self._starting_hand_probs[p_i][starting_hand] = prob / total_prob + # TODO: figure out typing for dicts.. def _update_hole_cards_bayes(self, offline_strategy: Dict): - """Get probability of reach for each pair of hole cards for each player""" + """Get probability of reach for each starting hand for each player""" n_players = len(self._table.players) player_indices: List[int] = [p_i for p_i in range(n_players)] for p_i in player_indices: + # TODO: might make since to put starting hands in the deck class for starting_hand in self._starting_hand_probs[p_i].keys(): + + starting_hand = list( + starting_hand + ) # TODO: is this bad? if "p_reach" in locals(): del p_reach action_sequence: Dict[str, List[str]] = collections.defaultdict(list) - previous_betting_stage = "pre_flop" - first_action_round = False for idx, betting_stage in enumerate(self._history.keys()): - # import ipdb; - # ipdb.set_trace() n_actions_round = len(self._history[betting_stage]) for i in range(n_actions_round): - # if i == 0: - # betting_stage = previous_betting_stage - # elif i == n_actions_round - 1: - # previous_betting_stage = betting_stage - # else: - # betting_stage = betting_round action = self._history[betting_stage][i] while action == 'skip': i += 1 # action sequences don't end in skip @@ -368,114 +357,113 @@ def _update_hole_cards_bayes(self, offline_strategy: Dict): ph = i % n_players if p_i != ph: prob_reach_all_hands = [] - num_hands = 0 for opp_starting_hand in self._starting_hand_probs[ p_i ].keys(): - # TODO: clean this up - public_evals = [ - c.eval_card - for c in self._public_information[betting_stage] - ] - if len(set(opp_starting_hand).union(set(public_evals)).union(set(starting_hand))) < \ - len(opp_starting_hand) + len(starting_hand) + len(public_evals): + opp_starting_hand = list( + opp_starting_hand + ) + publics = self._public_information[betting_stage] + if len( + set(opp_starting_hand).union( + set(publics) + ).union(set(starting_hand)) + ) < len( + opp_starting_hand + ) + len( + starting_hand + ) + len( + publics + ): prob = 0 - num_hands += 1 else: - num_hands += 1 - - public_cards = self._public_information[ + publics = self._public_information[ betting_stage ] - public_cards_evals = [c.eval_card for c in public_cards] infoset = self._info_set_helper( opp_starting_hand, - public_cards_evals, + publics, action_sequence, betting_stage, ) - # check to see if the strategy exists, if not equal probability - # TODO: is this hacky? problem with defaulting to 1 / 3, is that it - # doesn't work for calculations that need to be made with the object's values + # check to see if the strategy exists, + # if not equal probability + # TODO: is this overly hacky? + # Problem with defaulting to 1 / 3, is that it + # it doesn't work for calculations that + # need to be made with the object's values try: # TODO: with or without keys prob = offline_strategy[infoset][action] except KeyError: prob = 1 / len(self.legal_actions) prob_reach_all_hands.append(prob) - # import ipdb; - # ipdb.set_trace() - prob2 = sum(prob_reach_all_hands) / num_hands + total_opp_prob_h = sum(prob_reach_all_hands) /\ + len(prob_reach_all_hands) if "p_reach" not in locals(): - p_reach = prob2 + p_reach = total_opp_prob_h else: - p_reach *= prob2 + p_reach *= total_opp_prob_h elif p_i == ph: - public_evals = [ - c.eval_card - for c in self._public_information[betting_stage] - ] - if len(set(starting_hand).union(set(public_evals))) < ( - len(public_evals) + 2 + publics = self._public_information[betting_stage] + if len( + set(starting_hand).union( + set(publics) + ) + ) < ( + len(publics) + 2 ): - prob = 0 + total_prob = 0 else: - public_cards = self._public_information[betting_stage] - public_cards_evals = [c.eval_card for c in public_cards] + publics = self._public_information[betting_stage] infoset = self._info_set_helper( starting_hand, - public_cards_evals, + publics, action_sequence, betting_stage, ) # TODO: Check this try: - prob = offline_strategy[infoset][action] + total_prob = offline_strategy[infoset][action] except KeyError: - prob = 1 / len(self.legal_actions) + total_prob = 1 / len(self.legal_actions) if "p_reach" not in locals(): - p_reach = prob + p_reach = total_prob else: - p_reach *= prob + p_reach *= total_prob action_sequence[betting_stage].append(action) self._starting_hand_probs[p_i][tuple(starting_hand)] = p_reach self._normalize_bayes() - # TODO: delete this? at least for our purposes we don't need it again def deal_bayes(self): - start = time.time() + # TODO: Not sure if I need this yet lut = self.info_set_lut self.info_set_lut = {} new_state = copy.deepcopy(self) new_state.info_set_lut = self.info_set_lut = lut - end = time.time() - print(f"Took {start - end} to load") - players = list(range(len(self.players))) random.shuffle(players) - - # TODO should contain the current public cards/heros real hand, if exists - card_evals_selected = [] - - for player in players: - # does this maintain order? - starting_hand_eval = new_state._get_starting_hand(player) - len_union = len(set(starting_hand_eval).union(set(card_evals_selected))) - len_individual = len(starting_hand_eval) + len(card_evals_selected) + cards_selected = [] + # TODO: this might be made better by selecting the first player's + # cards, then normalizing the second and third, etc.. + for p_i in players: + starting_hand = new_state._get_starting_hand(p_i) + len_union = len(set(starting_hand).union(set(cards_selected))) + len_individual = len(starting_hand) + len(cards_selected) while len_union < len_individual: - starting_hand_eval = new_state._get_starting_hand(player) - len_union = len(set(starting_hand_eval).union(set(card_evals_selected))) - len_individual = len(starting_hand_eval) + len(card_evals_selected) - for card_eval in starting_hand_eval: - card = new_state._evals_to_cards[card_eval] - new_state.players[player].add_private_card(card) - card_evals_selected += starting_hand_eval - cards_selected = [new_state._evals_to_cards[c] for c in card_evals_selected] + starting_hand = new_state._get_starting_hand(p_i) + len_union = len(set(starting_hand).union(set(cards_selected))) + len_individual = len(starting_hand) + len(cards_selected) + # TODO: pull this into a helper method, maybe it should + # be in the dealer class.. + for card in starting_hand: + new_state.players[p_i].add_private_card(card) + cards_selected += starting_hand cards_selected += new_state._public_cards for card in cards_selected: new_state._table.dealer.deck.remove(card) return new_state - # TODO add convenience method to supply public cards + # TODO add convenience method to supply public cards def load_game_state(self, offline_strategy: Dict, action_sequence: list): """ @@ -498,23 +486,31 @@ def load_game_state(self, offline_strategy: Dict, action_sequence: list): def _get_starting_hand(self, player_idx: int): """Get starting hand based on probability of reach""" - starting_hand_idxs = list(range(len(self._starting_hand_probs[player_idx].keys()))) - starting_hands_probs = list(self._starting_hand_probs[player_idx].values()) - starting_hand_idx = np.random.choice(starting_hand_idxs, 1, p=starting_hands_probs)[0] - starting_hand = list(self._starting_hand_probs[player_idx].keys())[starting_hand_idx] + starting_hands = list(self._starting_hand_probs[player_idx].keys()) + # hacky for using tuples as keys + starting_hands_idxs = list(range(len(starting_hands))) + starting_hands_probs = list(self._starting_hand_probs[ + player_idx + ].values()) + starting_hand_idx = np.random.choice( + starting_hands_idxs, + 1, + p=starting_hands_probs + )[0] + starting_hand = list(starting_hands[starting_hand_idx]) return starting_hand def _initialize_starting_hands(self): """Dictionary of starting hands to store probabilities in""" assert self.betting_stage == "pre_flop" - # TODO: make this abstracted for n_players - starting_hand_probs = {0: {}, 1: {}, 2: {}} + starting_hand_probs = {} n_players = len(self.players) starting_hands = self._get_card_combos(2) for p_i in range(n_players): + starting_hand_probs[p_i] = {} for starting_hand in starting_hands: starting_hand_probs[p_i][ - tuple([c.eval_card for c in starting_hand]) + starting_hand ] = 1 return starting_hand_probs @@ -523,15 +519,23 @@ def _info_set_helper( ): # didn't want to combine this with the other, as we may want to modularize soon """Get the information set for the current player.""" - cards = sorted(hole_cards, reverse=True,) - cards += sorted(public_cards, reverse=True,) - eval_cards = tuple(cards) + cards = sorted( + hole_cards, + key=operator.attrgetter("eval_card"), + reverse=True, + ) + cards += sorted( + public_cards, + key=operator.attrgetter("eval_card"), + reverse=True, + ) + eval_cards = tuple([int(c) for c in cards]) try: cards_cluster = self.info_set_lut[betting_stage][eval_cards] except KeyError: if not self.info_set_lut: raise ValueError("Pickle luts must be loaded for info set.") - elif eval_cards not in self.info_set_lut[self._betting_stage]: + elif eval_cards not in self.info_set_lut[betting_stage]: raise ValueError("Cards {cards} not in pickle files.") else: raise ValueError("Unrecognised betting stage in pickle files.") @@ -548,7 +552,7 @@ def _info_set_helper( def _get_card_combos(self, num_cards): """Get combinations of cards""" - return list(combinations(self._poker_engine.table.dealer.deck._cards_in_deck, num_cards)) + return list(combinations(self.cards_in_deck, num_cards)) @property def community_cards(self) -> List[Card]: @@ -560,6 +564,11 @@ def private_hands(self) -> Dict[ShortDeckPokerPlayer, List[Card]]: """Return all private hands.""" return {p: p.cards for p in self.players} + @property + def cards_in_deck(self): + """Returns current cards in deck""" + return self._table.dealer.deck._cards_in_deck + @property def initial_regret(self) -> Dict[str, float]: """Returns the default regret for this state.""" @@ -590,11 +599,6 @@ def n_players_started_round(self) -> bool: """Return n_players that started the round.""" return self._n_players_started_round - # @property - # def first_move_of_current_round(self) -> bool: - # """Return boolfor first move of current round.""" - # return self._first_move_of_current_round - @property def player_i(self) -> int: """Get the index of the players turn it is.""" @@ -603,11 +607,11 @@ def player_i(self) -> int: @player_i.setter def player_i(self, _: Any): """Raise an error if player_i is set.""" - raise ValueError(f"The player_i property should not be set.") + raise ValueError("The player_i property should not be set.") @property def betting_round(self) -> int: - """Algorithm 1 of pluribus supp. material references betting_round.""" + """Return 0 indexed betting round""" try: betting_round = self._betting_stage_to_round[self._betting_stage] except KeyError: @@ -631,7 +635,7 @@ def info_set(self) -> str: key=operator.attrgetter("eval_card"), reverse=True, ) - eval_cards = tuple([card.eval_card for card in cards]) + eval_cards = tuple([int(card) for card in cards]) try: cards_cluster = self.info_set_lut[self._betting_stage][eval_cards] except KeyError: diff --git a/pluribus/poker/card.py b/pluribus/poker/card.py index 5fe30a61..a3fc6db7 100644 --- a/pluribus/poker/card.py +++ b/pluribus/poker/card.py @@ -74,6 +74,9 @@ def __eq__(self, other): def __ne__(self, other): return int(self) != int(other) + def __hash__(self): + return hash(int(self)) + @property def eval_card(self) -> EvaluationCard: """Return an `EvaluationCard` for use in the `Evaluator`.""" @@ -178,4 +181,3 @@ def from_dict(x: Dict[str, Union[int, str]]): if set(x) != {"rank", "suit"}: raise NotImplementedError(f"Unrecognised dict {x}") return Card(rank=x["rank"], suit=x["suit"]) - diff --git a/pluribus/poker/deck.py b/pluribus/poker/deck.py index a463058b..c6801105 100644 --- a/pluribus/poker/deck.py +++ b/pluribus/poker/deck.py @@ -64,10 +64,6 @@ def pick(self, random: bool = True) -> Card: def remove(self, card): """Remove a specific card from the deck""" - # TODO: is there any reason for them to be? - # Maybe better to assert it's not - c = card.eval_card - cards_in_deck = [x.eval_card for x in self._cards_in_deck] - if c in cards_in_deck: - index = cards_in_deck.index(c) - self._dealt_cards.append(self._cards_in_deck.pop(index)) + if card in self._cards_in_deck: + self._cards_in_deck.remove(card) + self._dealt_cards.append(card) diff --git a/research/test_methodology/RT.py b/research/test_methodology/RT.py index caf557e4..b89b5815 100644 --- a/research/test_methodology/RT.py +++ b/research/test_methodology/RT.py @@ -21,8 +21,7 @@ agent_output = train( agent1.offline_strategy, public_cards, action_sequence, 40, 6, 6, 3, 2, 6 ) # TODO: back to 50 - with open("realtime-strategy-refactor-game-state.pkl", "wb") as file: + with open("realtime-strategy-refactor-deck.pkl", "wb") as file: pickle.dump(agent_output, file) import ipdb - ipdb.set_trace() From 2bbb7e91b83afb713987201dd4b6fd994e6c29d6 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Tue, 19 May 2020 21:38:02 -0400 Subject: [PATCH 23/42] cleaning up some errors --- research/test_methodology/RT.py | 20 ++-- research/test_methodology/RT_cfr.py | 180 ++-------------------------- 2 files changed, 21 insertions(+), 179 deletions(-) diff --git a/research/test_methodology/RT.py b/research/test_methodology/RT.py index b89b5815..388f321e 100644 --- a/research/test_methodology/RT.py +++ b/research/test_methodology/RT.py @@ -1,27 +1,25 @@ +from typing import List + import dill as pickle -from RT_cfr import * -from pluribus.games.short_deck.state import * -from pluribus.games.short_deck.agent import * +from RT_cfr import train +from pluribus.games.short_deck.agent import TrainedAgent from pluribus.poker.card import Card if __name__ == "__main__": - # public_cards = [Card("ace", "spades"), Card("queen", "spades"), Card("queen", "hearts")] - public_cards = [] + # public_cards = [Card("ace", "spades"), Card("queen", "spades"), + # Card("queen", "hearts")] + public_cards: List[Card] = [] # we load a (trained) strategy agent1 = TrainedAgent("../blueprint_algo/results_2020_05_10_21_36_47_291425") - # sorta hacky, but I loaded the average strategy above, now I'm replacing with # the better strategy # offline_strategy = joblib.load('/Users/colin/Downloads/offline_strategy_285800.gz') - # print(sys.getsizeof(offline_strategy)) - # agent1.offline_strategy = offline_strategy - # print(sys.getsizeof(agent1.offline_strategy)) action_sequence = ["raise", "call", "call", "call", "call"] agent_output = train( agent1.offline_strategy, public_cards, action_sequence, 40, 6, 6, 3, 2, 6 - ) # TODO: back to 50 - with open("realtime-strategy-refactor-deck.pkl", "wb") as file: + ) + with open("testing2.pkl", "wb") as file: pickle.dump(agent_output, file) import ipdb ipdb.set_trace() diff --git a/research/test_methodology/RT_cfr.py b/research/test_methodology/RT_cfr.py index 72fad794..07d31d21 100644 --- a/research/test_methodology/RT_cfr.py +++ b/research/test_methodology/RT_cfr.py @@ -1,199 +1,41 @@ -""" -""" from __future__ import annotations import logging - -logging.basicConfig(filename="test.txt", level=logging.DEBUG) +import sys from tqdm import trange +import numpy as np from pluribus import utils -from pluribus.games.short_deck.state import * -from pluribus.games.short_deck.agent import * +from pluribus.games.short_deck.state import ShortDeckPokerState, new_game +from pluribus.games.short_deck.agent import Agent +sys.path.append('../blueprint_algo') +from blueprint_short_deck_poker import calculate_strategy, cfr, cfrp def update_strategy(agent: Agent, state: ShortDeckPokerState, ph_test_node: int): """ - - :param state: the game state - :param i: the player, i = 1 is always first to act and i = 2 is always second to act, but they take turns who - updates the strategy (only one strategy) - :return: nothing, updates action count in the strategy of actions chosen according to sigma, this simple choosing of - actions is what allows the algorithm to build up preference for one action over another in a given spot + Update strategy for test node only """ - logging.debug("UPDATE STRATEGY") - logging.debug("########") - - logging.debug(f"P(h): {state.player_i}") - logging.debug(f"Betting Round {state._betting_stage}") - logging.debug(f"Community Cards {state._table.community_cards}") - logging.debug(f"Player 0 hole cards: {state.players[0].cards}") - logging.debug(f"Player 1 hole cards: {state.players[1].cards}") - logging.debug(f"Player 2 hole cards: {state.players[2].cards}") - logging.debug(f"Betting Action Correct?: {state.players}") - - ph = state.player_i # this is always the case no matter what i is - + ph = state.player_i if ph == ph_test_node: - try: - I = state.info_set - except: - import ipdb - - ipdb.set_trace() + I = state.info_set # calculate regret - logging.debug(f"About to Calculate Strategy, Regret: {agent.regret[I]}") - logging.debug(f"Current regret: {agent.regret[I]}") sigma = calculate_strategy(agent.regret, I, state) - logging.debug(f"Calculated Strategy for {I}: {sigma[I]}") # choose an action based of sigma try: a = np.random.choice(list(sigma[I].keys()), 1, p=list(sigma[I].values()))[0] - logging.debug(f"ACTION SAMPLED: ph {state.player_i} ACTION: {a}") except ValueError: p = 1 / len(state.legal_actions) probabilities = np.full(len(state.legal_actions), p) a = np.random.choice(state.legal_actions, p=probabilities) sigma[I] = {action: p for action in state.legal_actions} - logging.debug(f"ACTION SAMPLED: ph {state.player_i} ACTION: {a}") # Increment the action counter. agent.strategy[I][a] += 1 - logging.debug(f"Updated Strategy for {I}: {agent.strategy[I]}") return else: return - -def calculate_strategy( - regret: Dict[str, Dict[str, float]], I: str, state: ShortDeckPokerState, -): - """ - - :param regret: dictionary of regrets, I is key, then each action at I, with values being regret - :param sigma: dictionary of strategy updated by regret, iteration is key, then I is key, then each action with prob - :param I: - :param state: the game state - :return: doesn't return anything, just updates sigma - """ - sigma = collections.defaultdict(lambda: collections.defaultdict(lambda: 1 / 3)) - rsum = sum([max(x, 0) for x in regret[I].values()]) - for a in state.legal_actions: - if rsum > 0: - sigma[I][a] = max(regret[I][a], 0) / rsum - else: - sigma[I][a] = 1 / len(state.legal_actions) - return sigma - - -def cfr(agent: Agent, state: ShortDeckPokerState, i: int, t: int) -> float: - """ - regular cfr algo - - :param state: the game state - :param i: player - :param t: iteration - :return: expected value for node for player i - """ - logging.debug("CFR") - logging.debug("########") - logging.debug(f"Iteration: {t}") - logging.debug(f"Player Set to Update Regret: {i}") - logging.debug(f"P(h): {state.player_i}") - logging.debug(f"P(h) Updating Regret? {state.player_i == i}") - logging.debug(f"Betting Round {state._betting_stage}") - logging.debug(f"Community Cards {state._table.community_cards}") - logging.debug(f"Player 0 hole cards: {state.players[0].cards}") - logging.debug(f"Player 1 hole cards: {state.players[1].cards}") - logging.debug(f"Player 2 hole cards: {state.players[2].cards}") - logging.debug(f"Betting Action Correct?: {state.players}") - - ph = state.player_i - - player_not_in_hand = not state.players[i].is_active - if state.is_terminal or player_not_in_hand: - return state.payout[i] - - # NOTE(fedden): The logic in Algorithm 1 in the supplementary material - # instructs the following lines of logic, but state class - # will already skip to the next in-hand player. - # elif p_i not in hand: - # cfr() - # NOTE(fedden): According to Algorithm 1 in the supplementary material, - # we would add in the following bit of logic. However we - # already have the game logic embedded in the state class, - # and this accounts for the chance samplings. In other words, - # it makes sure that chance actions such as dealing cards - # happen at the appropriate times. - # elif h is chance_node: - # sample action from strategy for h - # cfr() - - elif ph == i: - try: - I = state.info_set - except: - import ipdb - - ipdb.set_trace() - # calculate strategy - logging.debug(f"About to Calculate Strategy, Regret: {agent.regret[I]}") - logging.debug(f"Current regret: {agent.regret[I]}") - sigma = calculate_strategy(agent.regret, I, state) - logging.debug(f"Calculated Strategy for {I}: {sigma[I]}") - - vo = 0.0 - voa = {} - for a in state.legal_actions: - logging.debug( - f"ACTION TRAVERSED FOR REGRET: ph {state.player_i} ACTION: {a}" - ) - new_state: ShortDeckPokerState = state.apply_action(a) - voa[a] = cfr(agent, new_state, i, t) - logging.debug(f"Got EV for {a}: {voa[a]}") - vo += sigma[I][a] * voa[a] - logging.debug( - f"""Added to Node EV for ACTION: {a} INFOSET: {I} - STRATEGY: {sigma[I][a]}: {sigma[I][a] * voa[a]}""" - ) - logging.debug(f"Updated EV at {I}: {vo}") - - for a in state.legal_actions: - agent.regret[I][a] += voa[a] - vo - logging.debug(f"Updated Regret at {I}: {agent.regret[I]}") - - return vo - else: - # import ipdb; - # ipdb.set_trace() - try: - Iph = state.info_set - except: - import ipdb - - ipdb.set_trace() - logging.debug(f"About to Calculate Strategy, Regret: {agent.regret[Iph]}") - logging.debug(f"Current regret: {agent.regret[Iph]}") - sigma = calculate_strategy(agent.regret, Iph, state) - logging.debug(f"Calculated Strategy for {Iph}: {sigma[Iph]}") - - try: - a = np.random.choice( - list(sigma[Iph].keys()), 1, p=list(sigma[Iph].values()), - )[0] - logging.debug(f"ACTION SAMPLED: ph {state.player_i} ACTION: {a}") - - except ValueError: - p = 1 / len(state.legal_actions) - probabilities = np.full(len(state.legal_actions), p) - a = np.random.choice(state.legal_actions, p=probabilities) - sigma[Iph] = {action: p for action in state.legal_actions} - logging.debug(f"ACTION SAMPLED: ph {state.player_i} ACTION: {a}") - - new_state: ShortDeckPokerState = state.apply_action(a) - return cfr(agent, new_state, i, t) - - def train( offline_strategy: Dict, public_cards: list, @@ -206,10 +48,12 @@ def train( update_threshold: int, ): """Train agent.""" + # TODO: fix the seed utils.random.seed(36) agent = Agent() - state: ShortDeckPokerState = new_game(3, real_time_test=True, public_cards=public_cards) + state: ShortDeckPokerState = new_game(3, real_time_test=True, + public_cards=public_cards) current_game_state: ShortDeckPokerState = state.load_game_state( offline_strategy, action_sequence From 965139a2ebc6d601013fd299c0ea1e01772d849e Mon Sep 17 00:00:00 2001 From: big-c-note Date: Tue, 19 May 2020 23:58:37 -0400 Subject: [PATCH 24/42] reorganizing methods, info_set_builder takes args, remove unused attributes --- pluribus/games/short_deck/state.py | 239 ++++++++++++----------------- 1 file changed, 101 insertions(+), 138 deletions(-) diff --git a/pluribus/games/short_deck/state.py b/pluribus/games/short_deck/state.py index 4a763d5c..806dfd56 100644 --- a/pluribus/games/short_deck/state.py +++ b/pluribus/games/short_deck/state.py @@ -9,7 +9,6 @@ from typing import Any, Dict, List, Optional, Tuple from itertools import combinations import random -import time import dill as pickle import numpy as np @@ -90,12 +89,13 @@ def __init__( self._poker_engine.round_setup() # Deal private cards to players. if not self.real_time_test: - self._poker_engine.table.dealer.deal_private_cards(self._table.players) + self._poker_engine.table.dealer.deal_private_cards( + self._table.players + ) # Store the actions as they come in here. self._history: Dict[str, List[str]] = collections.defaultdict(list) self._public_information: Dict[str, List[Card]] = collections.defaultdict(list) self._betting_stage = "pre_flop" - self._previous_betting_stage = None self._betting_stage_to_round: Dict[str, int] = { "pre_flop": 0, "flop": 1, @@ -193,11 +193,6 @@ def apply_action(self, action_str: Optional[str]) -> ShortDeckPokerState: skip_actions = ["skip" for _ in range(new_state._skip_counter)] new_state._history[new_state.betting_stage] += skip_actions new_state._history[new_state.betting_stage].append(str(action)) - # save public information - # if new_state._first_move_of_current_round: - # new_state._public_information[ - # new_state.betting_stage - # ] = new_state._table.community_cards new_state._n_actions += 1 new_state._skip_counter = 0 # Player has made move, increment the player that is next. @@ -232,6 +227,55 @@ def apply_action(self, action_str: Optional[str]) -> ShortDeckPokerState: new_state.current_player.is_turn = True return new_state + def load_game_state(self, offline_strategy: Dict[str, Dict[str, float]], + action_sequence: list): + """ + Follow through the action sequence provided to get current node. + :param action_sequence: List of actions without 'skip' + """ + if not action_sequence: + # TODO: not 100 percent sure I need to deep copy + lut = self.info_set_lut + self.info_set_lut = {} + new_state = copy.deepcopy(self) + new_state.info_set_lut = self.info_set_lut = lut + new_state._update_hole_cards_bayes(offline_strategy) + return new_state + a = action_sequence.pop(0) + if a == "skip": + a = action_sequence.pop(0) + new_state = self.apply_action(a) + return new_state.load_game_state(offline_strategy, action_sequence) + + def deal_bayes(self): + # TODO: Not sure if I need this yet + lut = self.info_set_lut + self.info_set_lut = {} + new_state = copy.deepcopy(self) + new_state.info_set_lut = self.info_set_lut = lut + players = list(range(len(self.players))) + random.shuffle(players) + cards_selected = [] + # TODO: this might be made better by selecting the first player's + # cards, then normalizing the second and third, etc.. + for p_i in players: + starting_hand = new_state._get_starting_hand(p_i) + len_union = len(set(starting_hand).union(set(cards_selected))) + len_individual = len(starting_hand) + len(cards_selected) + while len_union < len_individual: + starting_hand = new_state._get_starting_hand(p_i) + len_union = len(set(starting_hand).union(set(cards_selected))) + len_individual = len(starting_hand) + len(cards_selected) + # TODO: pull this into a helper method, maybe it should + # be in the dealer class.. + for card in starting_hand: + new_state.players[p_i].add_private_card(card) + cards_selected += starting_hand + cards_selected += new_state._public_cards + for card in cards_selected: + new_state._table.dealer.deck.remove(card) + return new_state + @staticmethod def load_pickle_files(pickle_dir: str) -> Dict[str, Dict[Tuple[int, ...], str]]: """Load pickle files into memory.""" @@ -277,7 +321,6 @@ def _increment_stage(self): if self._betting_stage == "pre_flop": # Progress from private cards to the flop. self._betting_stage = "flop" - self._previous_betting_stage = "pre_flop" if len(self._public_cards) >= 3: community_cards = self._public_cards[:3] self._poker_engine.table.community_cards += community_cards @@ -289,7 +332,6 @@ def _increment_stage(self): elif self._betting_stage == "flop": # Progress from flop to turn. self._betting_stage = "turn" - self._previous_betting_stage = "flop" if len(self._public_cards) >= 4: community_cards = self._public_cards[3:4] self._poker_engine.table.community_cards += community_cards @@ -301,7 +343,6 @@ def _increment_stage(self): elif self._betting_stage == "turn": # Progress from turn to river. self._betting_stage = "river" - self._previous_betting_stage = "turn" if len(self._public_cards) == 5: community_cards = self._public_cards[4:] self._poker_engine.table.community_cards += community_cards @@ -313,12 +354,30 @@ def _increment_stage(self): elif self._betting_stage == "river": # Progress to the showdown. self._betting_stage = "show_down" - self._previous_betting_stage = "river" elif self._betting_stage in {"show_down", "terminal"}: pass else: raise ValueError(f"Unknown betting_stage: {self._betting_stage}") + def _initialize_starting_hands(self) -> Dict[int, Dict[List[Card], float]]: + """Dictionary of starting hands to store probabilities in""" + assert self.betting_stage == "pre_flop" + starting_hand_probs: Dict = {} + n_players = len(self.players) + starting_hands = self._get_card_combos(2) + for p_i in range(n_players): + starting_hand_probs[p_i] = {} + for starting_hand in starting_hands: + starting_hand_probs[p_i][ + starting_hand + ] = 1 + return starting_hand_probs + + + def _get_card_combos(self, num_cards) -> List[Tuple[Any, ...]]: + """Get combinations of cards""" + return list(combinations(self.cards_in_deck, num_cards)) + def _normalize_bayes(self): """Normalize probability of reach for each player""" n_players = len(self.players) @@ -327,15 +386,14 @@ def _normalize_bayes(self): for starting_hand, prob in self._starting_hand_probs[p_i].items(): self._starting_hand_probs[p_i][starting_hand] = prob / total_prob - # TODO: figure out typing for dicts.. - def _update_hole_cards_bayes(self, offline_strategy: Dict): + def _update_hole_cards_bayes(self, offline_strategy: Dict[str, Dict[str, + float]]): """Get probability of reach for each starting hand for each player""" n_players = len(self._table.players) player_indices: List[int] = [p_i for p_i in range(n_players)] for p_i in player_indices: # TODO: might make since to put starting hands in the deck class for starting_hand in self._starting_hand_probs[p_i].keys(): - starting_hand = list( starting_hand ) @@ -380,11 +438,11 @@ def _update_hole_cards_bayes(self, offline_strategy: Dict): publics = self._public_information[ betting_stage ] - infoset = self._info_set_helper( - opp_starting_hand, - publics, - action_sequence, - betting_stage, + infoset = self._info_set_builder( + hole_cards=opp_starting_hand, + public_cards=publics, + history=action_sequence, + this_betting_stage=betting_stage, ) # check to see if the strategy exists, # if not equal probability @@ -416,11 +474,11 @@ def _update_hole_cards_bayes(self, offline_strategy: Dict): total_prob = 0 else: publics = self._public_information[betting_stage] - infoset = self._info_set_helper( - starting_hand, - publics, - action_sequence, - betting_stage, + infoset = self._info_set_builder( + hole_cards=starting_hand, + public_cards=publics, + history=action_sequence, + this_betting_stage=betting_stage, ) # TODO: Check this try: @@ -435,56 +493,7 @@ def _update_hole_cards_bayes(self, offline_strategy: Dict): self._starting_hand_probs[p_i][tuple(starting_hand)] = p_reach self._normalize_bayes() - def deal_bayes(self): - # TODO: Not sure if I need this yet - lut = self.info_set_lut - self.info_set_lut = {} - new_state = copy.deepcopy(self) - new_state.info_set_lut = self.info_set_lut = lut - players = list(range(len(self.players))) - random.shuffle(players) - cards_selected = [] - # TODO: this might be made better by selecting the first player's - # cards, then normalizing the second and third, etc.. - for p_i in players: - starting_hand = new_state._get_starting_hand(p_i) - len_union = len(set(starting_hand).union(set(cards_selected))) - len_individual = len(starting_hand) + len(cards_selected) - while len_union < len_individual: - starting_hand = new_state._get_starting_hand(p_i) - len_union = len(set(starting_hand).union(set(cards_selected))) - len_individual = len(starting_hand) + len(cards_selected) - # TODO: pull this into a helper method, maybe it should - # be in the dealer class.. - for card in starting_hand: - new_state.players[p_i].add_private_card(card) - cards_selected += starting_hand - cards_selected += new_state._public_cards - for card in cards_selected: - new_state._table.dealer.deck.remove(card) - return new_state - # TODO add convenience method to supply public cards - - def load_game_state(self, offline_strategy: Dict, action_sequence: list): - """ - Follow through the action sequence provided to get current node. - :param action_sequence: List of actions without 'skip' - """ - if not action_sequence: - # TODO: not 100 percent sure I need to deep copy - lut = self.info_set_lut - self.info_set_lut = {} - new_state = copy.deepcopy(self) - new_state.info_set_lut = self.info_set_lut = lut - new_state._update_hole_cards_bayes(offline_strategy) - return new_state - a = action_sequence.pop(0) - if a == "skip": - a = action_sequence.pop(0) - new_state = self.apply_action(a) - return new_state.load_game_state(offline_strategy, action_sequence) - - def _get_starting_hand(self, player_idx: int): + def _get_starting_hand(self, player_idx: int) -> List[Card]: """Get starting hand based on probability of reach""" starting_hands = list(self._starting_hand_probs[player_idx].keys()) # hacky for using tuples as keys @@ -500,25 +509,17 @@ def _get_starting_hand(self, player_idx: int): starting_hand = list(starting_hands[starting_hand_idx]) return starting_hand - def _initialize_starting_hands(self): - """Dictionary of starting hands to store probabilities in""" - assert self.betting_stage == "pre_flop" - starting_hand_probs = {} - n_players = len(self.players) - starting_hands = self._get_card_combos(2) - for p_i in range(n_players): - starting_hand_probs[p_i] = {} - for starting_hand in starting_hands: - starting_hand_probs[p_i][ - starting_hand - ] = 1 - return starting_hand_probs - - def _info_set_helper( - self, hole_cards, public_cards, action_sequence, betting_stage - ): - # didn't want to combine this with the other, as we may want to modularize soon + def _info_set_builder(self, hole_cards=None, public_cards=None, + history=None, this_betting_stage=None) -> str: """Get the information set for the current player.""" + if hole_cards is None: + hole_cards = self.current_player.cards + if public_cards is None: + public_cards = self._table.community_cards + if history is None: + history = self._history + if this_betting_stage is None: + this_betting_stage = self._betting_stage cards = sorted( hole_cards, key=operator.attrgetter("eval_card"), @@ -529,31 +530,24 @@ def _info_set_helper( key=operator.attrgetter("eval_card"), reverse=True, ) - eval_cards = tuple([int(c) for c in cards]) + eval_cards = tuple([int(card) for card in cards]) try: - cards_cluster = self.info_set_lut[betting_stage][eval_cards] + cards_cluster = self.info_set_lut[this_betting_stage][eval_cards] except KeyError: - if not self.info_set_lut: - raise ValueError("Pickle luts must be loaded for info set.") - elif eval_cards not in self.info_set_lut[betting_stage]: - raise ValueError("Cards {cards} not in pickle files.") - else: - raise ValueError("Unrecognised betting stage in pickle files.") + return "default info set, please ensure you load it correctly" + # Convert history from a dict of lists to a list of dicts as I'm + # paranoid about JSON's lack of care with insertion order. info_set_dict = { "cards_cluster": cards_cluster, "history": [ {betting_stage: [str(action) for action in actions]} - for betting_stage, actions in action_sequence.items() + for betting_stage, actions in history.items() ], } return json.dumps( info_set_dict, separators=(",", ":"), cls=utils.io.NumpyJSONEncoder ) - def _get_card_combos(self, num_cards): - """Get combinations of cards""" - return list(combinations(self.cards_in_deck, num_cards)) - @property def community_cards(self) -> List[Card]: """Return all shared/public cards.""" @@ -584,11 +578,6 @@ def betting_stage(self) -> str: """Return betting stage.""" return self._betting_stage - @property - def previous_betting_stage(self) -> str: - """Return previous betting stage.""" - return self._previous_betting_stage - @property def all_players_have_actioned(self) -> bool: """Return whether all players have made atleast one action.""" @@ -625,33 +614,7 @@ def betting_round(self) -> int: @property def info_set(self) -> str: """Get the information set for the current player.""" - cards = sorted( - self.current_player.cards, - key=operator.attrgetter("eval_card"), - reverse=True, - ) - cards += sorted( - self._table.community_cards, - key=operator.attrgetter("eval_card"), - reverse=True, - ) - eval_cards = tuple([int(card) for card in cards]) - try: - cards_cluster = self.info_set_lut[self._betting_stage][eval_cards] - except KeyError: - return "default info set, please ensure you load it correctly" - # Convert history from a dict of lists to a list of dicts as I'm - # paranoid about JSON's lack of care with insertion order. - info_set_dict = { - "cards_cluster": cards_cluster, - "history": [ - {betting_stage: [str(action) for action in actions]} - for betting_stage, actions in self._history.items() - ], - } - return json.dumps( - info_set_dict, separators=(",", ":"), cls=utils.io.NumpyJSONEncoder - ) + return self._info_set_builder() @property def payout(self) -> Dict[int, int]: From 4f237a919557f6dd17b529dbe3275cb56b111114 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Sat, 23 May 2020 18:22:56 -0400 Subject: [PATCH 25/42] making unnormalized strategy default, calculate strategy based on temporary regret, only weight temp regret --- pluribus/games/short_deck/agent.py | 21 ++- pluribus/games/short_deck/state.py | 8 +- research/test_methodology/RT.py | 13 +- research/test_methodology/RT_cfr.py | 176 ++++++++++++++++-- .../average_unnormalized_strategy.py | 92 +++++++++ 5 files changed, 284 insertions(+), 26 deletions(-) create mode 100644 research/test_methodology/average_unnormalized_strategy.py diff --git a/pluribus/games/short_deck/agent.py b/pluribus/games/short_deck/agent.py index ce924831..c086a433 100644 --- a/pluribus/games/short_deck/agent.py +++ b/pluribus/games/short_deck/agent.py @@ -12,14 +12,31 @@ class Agent: # action. This made it easier to unprune actions that were # initially pruned but later improved. This also prevented # integer overflows". - def __init__(self): + def __init__(self, regret_dir=None): self.strategy = collections.defaultdict( lambda: collections.defaultdict(lambda: 0) ) - self.regret = collections.defaultdict( + if regret_dir: + offline_strategy = joblib.load(regret_dir) + self.regret = collections.defaultdict( + lambda: collections.defaultdict(lambda: 0), + offline_strategy['regret'] + ) + else: + self.regret = collections.defaultdict( + lambda: collections.defaultdict(lambda: 0) + ) + self.tmp_regret = collections.defaultdict( lambda: collections.defaultdict(lambda: 0) ) + def reset_new_regret(self): + """Remove regret from temporary storage""" + del self.tmp_regret + self.tmp_regret = collections.defaultdict( + lambda: collections.defaultdict(lambda: 0) + ) + # TODO: Change to Leon's newest iteration on this method class TrainedAgent(Agent): diff --git a/pluribus/games/short_deck/state.py b/pluribus/games/short_deck/state.py index 806dfd56..43b2e7ed 100644 --- a/pluribus/games/short_deck/state.py +++ b/pluribus/games/short_deck/state.py @@ -387,7 +387,7 @@ def _normalize_bayes(self): self._starting_hand_probs[p_i][starting_hand] = prob / total_prob def _update_hole_cards_bayes(self, offline_strategy: Dict[str, Dict[str, - float]]): + float]]): """Get probability of reach for each starting hand for each player""" n_players = len(self._table.players) player_indices: List[int] = [p_i for p_i in range(n_players)] @@ -453,6 +453,9 @@ def _update_hole_cards_bayes(self, offline_strategy: Dict[str, Dict[str, try: # TODO: with or without keys prob = offline_strategy[infoset][action] + # Normalizing since offline_stregy is not + prob /= sum(offline_strategy[infoset]\ + .values()) except KeyError: prob = 1 / len(self.legal_actions) prob_reach_all_hands.append(prob) @@ -483,6 +486,9 @@ def _update_hole_cards_bayes(self, offline_strategy: Dict[str, Dict[str, # TODO: Check this try: total_prob = offline_strategy[infoset][action] + # Normalizing since offline_stregy is not + total_prob /= sum(offline_strategy[infoset]\ + .values()) except KeyError: total_prob = 1 / len(self.legal_actions) if "p_reach" not in locals(): diff --git a/research/test_methodology/RT.py b/research/test_methodology/RT.py index 388f321e..b79adbb4 100644 --- a/research/test_methodology/RT.py +++ b/research/test_methodology/RT.py @@ -1,10 +1,11 @@ from typing import List +import joblib import dill as pickle from RT_cfr import train -from pluribus.games.short_deck.agent import TrainedAgent from pluribus.poker.card import Card +from pluribus.games.short_deck.agent import Agent if __name__ == "__main__": @@ -12,12 +13,12 @@ # Card("queen", "hearts")] public_cards: List[Card] = [] # we load a (trained) strategy - agent1 = TrainedAgent("../blueprint_algo/results_2020_05_10_21_36_47_291425") - # the better strategy - # offline_strategy = joblib.load('/Users/colin/Downloads/offline_strategy_285800.gz') + agent = Agent(regret_dir='test_strategy/strategy_100.gz') action_sequence = ["raise", "call", "call", "call", "call"] - agent_output = train( - agent1.offline_strategy, public_cards, action_sequence, 40, 6, 6, 3, 2, 6 + agent_output, offline_strategy = train( + 'test_strategy/unnormalized_output/offline_strategy_100.gz', + 'test_strategy/strategy_100.gz', public_cards, action_sequence, + 40, 6, 6, 3, 2, 6, 10 ) with open("testing2.pkl", "wb") as file: pickle.dump(agent_output, file) diff --git a/research/test_methodology/RT_cfr.py b/research/test_methodology/RT_cfr.py index 07d31d21..a4bfd363 100644 --- a/research/test_methodology/RT_cfr.py +++ b/research/test_methodology/RT_cfr.py @@ -2,6 +2,11 @@ import logging import sys +import collections +from typing import Dict +import copy +import json +import joblib from tqdm import trange import numpy as np @@ -9,8 +14,36 @@ from pluribus import utils from pluribus.games.short_deck.state import ShortDeckPokerState, new_game from pluribus.games.short_deck.agent import Agent -sys.path.append('../blueprint_algo') -from blueprint_short_deck_poker import calculate_strategy, cfr, cfrp + +sys.path.append("../blueprint_algo") + + + + +def to_dict(**kwargs) -> Dict[str, Any]: + """Hacky method to convert weird collections dicts to regular dicts.""" + return json.loads(json.dumps(copy.deepcopy(kwargs))) + + +def normalize_strategy(this_info_sets_regret: Dict[str, float]) -> Dict[str, float]: + """Calculate the strategy based on the current information sets regret.""" + # TODO: Could we instanciate a state object from an info set? + actions = this_info_sets_regret.keys() + regret_sum = sum([max(regret, 0) for regret in this_info_sets_regret.values()]) + if regret_sum > 0: + strategy: Dict[str, float] = { + action: max(this_info_sets_regret[action], 0) / regret_sum + for action in actions + } + elif this_info_sets_regret == {}: + # Don't return strategy if no strategy was made + # during training + strategy: Dict[str, float] = {} + elif regret_sum == 0: + # Regret is negative, we learned something + default_probability = 1 / len(actions) + strategy: Dict[str, float] = {action: default_probability for action in actions} + return strategy def update_strategy(agent: Agent, state: ShortDeckPokerState, ph_test_node: int): @@ -21,7 +54,8 @@ def update_strategy(agent: Agent, state: ShortDeckPokerState, ph_test_node: int) if ph == ph_test_node: I = state.info_set # calculate regret - sigma = calculate_strategy(agent.regret, I, state) + sigma = calculate_strategy(agent.regret, I, state, offline_strategy, + unnormalized_strategy) # choose an action based of sigma try: a = np.random.choice(list(sigma[I].keys()), 1, p=list(sigma[I].values()))[0] @@ -36,8 +70,101 @@ def update_strategy(agent: Agent, state: ShortDeckPokerState, ph_test_node: int) else: return + +def calculate_strategy( + regret: Dict[str, Dict[str, float]], + I: str, + state: ShortDeckPokerState, +): + """ + + :param regret: dictionary of regrets, I is key, then each action at I, with values being regret + :param sigma: dictionary of strategy updated by regret, iteration is key, then I is key, then each action with prob + :param I: + :param state: the game state + :return: doesn't return anything, just updates sigma + """ + sigma = collections.defaultdict(lambda: collections.defaultdict(lambda: 1 / 3)) + rsum = sum([max(x, 0) for x in regret[I].values()]) + for a in state.legal_actions: + if rsum > 0: + sigma[I][a] = max(regret[I][a], 0) / rsum + else: + sigma[I][a] = 1 / len(state.legal_actions) + return sigma + + +def cfr(agent: Agent, state: ShortDeckPokerState, i: int, t: int) -> float: + """ + regular cfr algo + + :param state: the game state + :param i: player + :param t: iteration + :return: expected value for node for player i + """ + ph = state.player_i + + player_not_in_hand = not state.players[i].is_active + if state.is_terminal or player_not_in_hand: + return state.payout[i] + + # NOTE(fedden): The logic in Algorithm 1 in the supplementary material + # instructs the following lines of logic, but state class + # will already skip to the next in-hand player. + # elif p_i not in hand: + # cfr() + # NOTE(fedden): According to Algorithm 1 in the supplementary material, + # we would add in the following bit of logic. However we + # already have the game logic embedded in the state class, + # and this accounts for the chance samplings. In other words, + # it makes sure that chance actions such as dealing cards + # happen at the appropriate times. + # elif h is chance_node: + # sample action from strategy for h + # cfr() + + elif ph == i: + I = state.info_set + # calculate strategy + if agent.tmp_regret[I] == {}: + agent.tmp_regret[I] == agent.regret[I].copy() + sigma = calculate_strategy(agent.tmp_regret, I, state) + + vo = 0.0 + voa = {} + for a in state.legal_actions: + new_state: ShortDeckPokerState = state.apply_action(a) + voa[a] = cfr(agent, new_state, i, t) + vo += sigma[I][a] * voa[a] + + for a in state.legal_actions: + agent.tmp_regret[I][a] += voa[a] - vo + + return vo + else: + Iph = state.info_set + if agent.tmp_regret[Iph] == {}: + agent.tmp_regret[Iph] == agent.regret[Iph].copy() + sigma = calculate_strategy(agent.regret, Iph, state) + + try: + a = np.random.choice( + list(sigma[Iph].keys()), 1, p=list(sigma[Iph].values()), + )[0] + except KeyError: + p = 1 / len(state.legal_actions) + probabilities = np.full(len(state.legal_actions), p) + a = np.random.choice(state.legal_actions, p=probabilities) + sigma[Iph] = {action: p for action in state.legal_actions} + + new_state: ShortDeckPokerState = state.apply_action(a) + return cfr(agent, new_state, i, t) + + def train( - offline_strategy: Dict, + offline_strategy_path: str, + regret_path: str, public_cards: list, action_sequence: list, n_iterations: int, @@ -46,20 +173,22 @@ def train( n_players: int, update_interval: int, update_threshold: int, + dump_int: int, ): """Train agent.""" # TODO: fix the seed utils.random.seed(36) - agent = Agent() + agent = Agent(regret_dir=regret_path) - state: ShortDeckPokerState = new_game(3, real_time_test=True, - public_cards=public_cards) + offline_strategy = joblib.load(offline_strategy_path) + state: ShortDeckPokerState = new_game( + 3, real_time_test=True, public_cards=public_cards + ) current_game_state: ShortDeckPokerState = state.load_game_state( - offline_strategy, - action_sequence + offline_strategy, action_sequence ) del offline_strategy - ph_test_node = current_game_state.player_i + # ph_test_node = current_game_state.player_i for t in trange(1, n_iterations + 1, desc="train iter"): print(t) if t == 2: @@ -67,14 +196,27 @@ def train( for i in range(n_players): # fixed position i # Create a new state. state: ShortDeckPokerState = current_game_state.deal_bayes() - if t % update_interval == 0 and t > update_threshold: - update_strategy(agent, state, ph_test_node) +# if t % update_interval == 0 and t > update_threshold: +# update_strategy(agent, state, ph_test_node) cfr(agent, state, i, t) if t < lcfr_threshold & t % discount_interval == 0: d = (t / discount_interval) / ((t / discount_interval) + 1) - for I in agent.regret.keys(): - for a in agent.regret[I].keys(): - agent.regret[I][a] *= d - agent.strategy[I][a] *= d + for I in agent.tmp_regret.keys(): + for a in agent.tmp_regret[I].keys(): + agent.tmp_regret[I][a] *= d + + offline_strategy = joblib.load(offline_strategy_path) + # Adding the regret back to the regret dict + for I in agent.tmp_regret.keys(): + if agent.tmp_regret != {}: + agent.regret[I] = agent.tmp_regret[I].copy() - return agent + # Add the unnormalized strategy into the original + for info_set, this_info_sets_regret in sorted(agent.tmp_regret.items()): + # If this_info_sets_regret == {}, we do nothing + strategy = normalize_strategy(this_info_sets_regret) + if info_set not in offline_strategy: + offline_strategy[info_set] = {a: 0 for a in strategy.keys()} + for action, probability in strategy.items(): + offline_strategy[info_set][action] += t / dump_int * probability + return agent, offline_strategy diff --git a/research/test_methodology/average_unnormalized_strategy.py b/research/test_methodology/average_unnormalized_strategy.py new file mode 100644 index 00000000..08cf665f --- /dev/null +++ b/research/test_methodology/average_unnormalized_strategy.py @@ -0,0 +1,92 @@ +import collections +import glob +import os +import re +from typing import Dict, List, Union + +import click +import joblib +from tqdm import tqdm + + +def calculate_strategy(this_info_sets_regret: Dict[str, float]) -> Dict[str, float]: + """Calculate the strategy based on the current information sets regret.""" + # TODO: Could we instanciate a state object from an info set? + actions = this_info_sets_regret.keys() + regret_sum = sum([max(regret, 0) for regret in this_info_sets_regret.values()]) + if regret_sum > 0: + strategy: Dict[str, float] = { + action: max(this_info_sets_regret[action], 0) / regret_sum + for action in actions + } + elif this_info_sets_regret == {}: + # Don't return strategy if no strategy was made + # during training + strategy: Dict[str, float] = {} + elif regret_sum == 0: + # Regret is negative, we learned something + default_probability = 1 / len(actions) + strategy: Dict[str, float] = {action: default_probability for action in actions} + return strategy + + +def try_to_int(text: str) -> Union[str, int]: + """Attempt to return int.""" + return int(text) if text.isdigit() else text + + +def natural_key(text): + """Sort with natural numbers.""" + return [try_to_int(c) for c in re.split(r"(\d+)", text)] + + +def average_strategy(all_file_paths: List[str]) -> Dict[str, Dict[str, float]]: + """Compute the mean strategy over all timesteps.""" + # The offline strategy for all information sets. + offline_strategy: Dict[str, Dict[str, float]] = collections.defaultdict( + lambda: collections.defaultdict(lambda: 0.0) + ) + # Sum up all strategies. + for dump_path in tqdm(all_file_paths, desc="loading dumps"): + # Load file. + try: + agent = joblib.load(dump_path) + except Exception as e: + tqdm.write(f"Failed to load file at {dump_path} because:{e}") + agent = {} + regret = agent.get("regret", {}) + # Sum probabilities from computed strategy.. + for info_set, this_info_sets_regret in sorted(regret.items()): + strategy = calculate_strategy(this_info_sets_regret) + # If strategy == {}, we do nothing + for action, probability in strategy.items(): + offline_strategy[info_set][action] += probability + # Return regular dict, not defaultdict. + return {info_set: dict(strategy) for info_set, strategy in offline_strategy.items()} + + +@click.command() +@click.option( + "--results_dir_path", default=".", help="the location of the agent file dumps." +) +@click.option( + "--write_dir_path", default=".", help="where to save the offline strategy" +) +def cli(results_dir_path: str, write_dir_path: str): + """Compute the strategy and write to file.""" + # Find all files to load. + all_file_paths = glob.glob(os.path.join(results_dir_path, "*.gz")) + if not all_file_paths: + raise ValueError(f"No agent dumps could be found at: {results_dir_path}") + # Sort the file paths in the order they were created. + all_file_paths = sorted(all_file_paths, key=natural_key) + offline_strategy = average_strategy(all_file_paths) + # Save dictionary to compressed file. + latest_file = os.path.basename(all_file_paths[-1]) + latest_iteration: int = int(re.findall(r"\d+", latest_file)[0]) + save_file: str = f"offline_strategy_{latest_iteration}.gz" + joblib.dump(offline_strategy, os.path.join(write_dir_path, save_file)) + + +if __name__ == "__main__": + cli() From e68efebc48f336711d700eaeba6170de4fb10392 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Sat, 23 May 2020 22:12:33 -0400 Subject: [PATCH 26/42] cleaning up code and testing a few configs --- pluribus/games/short_deck/agent.py | 93 +------------------ pluribus/games/short_deck/state.py | 30 +++--- research/test_methodology/RT.py | 17 ++-- research/test_methodology/RT_cfr.py | 92 +++--------------- .../average_unnormalized_strategy.py | 1 - 5 files changed, 38 insertions(+), 195 deletions(-) diff --git a/pluribus/games/short_deck/agent.py b/pluribus/games/short_deck/agent.py index c086a433..c55ab503 100644 --- a/pluribus/games/short_deck/agent.py +++ b/pluribus/games/short_deck/agent.py @@ -1,23 +1,14 @@ -import os import collections -from typing import Dict import joblib class Agent: - # TODO(fedden): Note from the supplementary material, the data here will - # need to be lower precision: "To save memory, regrets were - # stored using 4-byte integers rather than 8-byte doubles. - # There was also a floor on regret at -310,000,000 for every - # action. This made it easier to unprune actions that were - # initially pruned but later improved. This also prevented - # integer overflows". - def __init__(self, regret_dir=None): + def __init__(self, regret_path=None): self.strategy = collections.defaultdict( lambda: collections.defaultdict(lambda: 0) ) - if regret_dir: - offline_strategy = joblib.load(regret_dir) + if regret_path: + offline_strategy = joblib.load(regret_path) self.regret = collections.defaultdict( lambda: collections.defaultdict(lambda: 0), offline_strategy['regret'] @@ -36,81 +27,3 @@ def reset_new_regret(self): self.tmp_regret = collections.defaultdict( lambda: collections.defaultdict(lambda: 0) ) - - -# TODO: Change to Leon's newest iteration on this method -class TrainedAgent(Agent): - """ - Agent who has been trained - Points to a folder whose strategies is calculated from regret and then averaged - """ - - def __init__(self, directory: str): - super().__init__() - self.offline_strategy = self._load_regret(directory) - - # TODO: the following could use a refactor, just getting through this - # rather quickly - def _calculate_strategy( - self, - regret: Dict[str, Dict[str, float]], - sigma: Dict[int, Dict[str, Dict[str, float]]], - I: str, - ): - """ - Get strategy from regret - """ - rsum = sum([max(x, 0) for x in regret[I].values()]) - ACTIONS = regret[I].keys() # TODO: this is hacky, might be a better way - for a in ACTIONS: - if rsum > 0: - sigma[I][a] = max(regret[I][a], 0) / rsum - else: - sigma[I][a] = 1 / len(ACTIONS) - return sigma - - def _average_strategy(self, directory: str): - files = [ - x - for x in os.listdir(directory) - if os.path.isfile(os.path.join(directory, x)) - ] - - offline_strategy: Dict = collections.defaultdict( - lambda: collections.defaultdict(lambda: 0) - ) - strategy_tmp = collections.defaultdict( - lambda: collections.defaultdict(lambda: 0) - ) - - for idx, f in enumerate(files): - if f in ["config.yaml", "strategy.gz"]: - continue - - regret_dict = joblib.load(directory + "/" + f)["regret"] - sigma = collections.defaultdict( - lambda: collections.defaultdict(lambda: 1 / 3) - ) - - for info_set, regret in sorted(regret_dict.items()): - sigma = self._calculate_strategy(regret_dict, sigma, info_set) - - for info_set, strategy in sigma.items(): - for action, probability in strategy.items(): - try: - strategy_tmp[info_set][action] += probability - except KeyError: - strategy_tmp[info_set][action] = probability - - for info_set, strategy in sorted(strategy_tmp.items()): - norm = sum(list(strategy.values())) - for action, probability in strategy.items(): - try: - offline_strategy[info_set][action] += probability / norm - except KeyError: - offline_strategy[info_set][action] = probability / norm - - return offline_strategy - - def _load_regret(self, directory: str): - return self._average_strategy(directory) diff --git a/pluribus/games/short_deck/state.py b/pluribus/games/short_deck/state.py index 43b2e7ed..fc5e0db7 100644 --- a/pluribus/games/short_deck/state.py +++ b/pluribus/games/short_deck/state.py @@ -163,7 +163,6 @@ def apply_action(self, action_str: Optional[str]) -> ShortDeckPokerState: new_state.info_set_lut = self.info_set_lut = lut # An action has been made, so alas we are not in the first move of the # current betting round. - # new_state._first_move_of_current_round = False if action_str is None: # Assert active player has folded already. assert ( @@ -207,7 +206,6 @@ def apply_action(self, action_str: Optional[str]) -> ShortDeckPokerState: # stage of the game. new_state._increment_stage() new_state._reset_betting_round_state() - # new_state._first_move_of_current_round = True if not new_state.current_player.is_active: new_state._skip_counter += 1 assert not new_state.current_player.is_active @@ -234,7 +232,7 @@ def load_game_state(self, offline_strategy: Dict[str, Dict[str, float]], :param action_sequence: List of actions without 'skip' """ if not action_sequence: - # TODO: not 100 percent sure I need to deep copy + # TODO: Not sure if I need to deepcopy lut = self.info_set_lut self.info_set_lut = {} new_state = copy.deepcopy(self) @@ -248,7 +246,7 @@ def load_game_state(self, offline_strategy: Dict[str, Dict[str, float]], return new_state.load_game_state(offline_strategy, action_sequence) def deal_bayes(self): - # TODO: Not sure if I need this yet + # TODO: Not sure if I need to deepcopy lut = self.info_set_lut self.info_set_lut = {} new_state = copy.deepcopy(self) @@ -256,7 +254,7 @@ def deal_bayes(self): players = list(range(len(self.players))) random.shuffle(players) cards_selected = [] - # TODO: this might be made better by selecting the first player's + # TODO: This would be better by selecting the first player's # cards, then normalizing the second and third, etc.. for p_i in players: starting_hand = new_state._get_starting_hand(p_i) @@ -373,7 +371,6 @@ def _initialize_starting_hands(self) -> Dict[int, Dict[List[Card], float]]: ] = 1 return starting_hand_probs - def _get_card_combos(self, num_cards) -> List[Tuple[Any, ...]]: """Get combinations of cards""" return list(combinations(self.cards_in_deck, num_cards)) @@ -387,17 +384,17 @@ def _normalize_bayes(self): self._starting_hand_probs[p_i][starting_hand] = prob / total_prob def _update_hole_cards_bayes(self, offline_strategy: Dict[str, Dict[str, - float]]): + float]]): """Get probability of reach for each starting hand for each player""" n_players = len(self._table.players) player_indices: List[int] = [p_i for p_i in range(n_players)] for p_i in player_indices: - # TODO: might make since to put starting hands in the deck class + # TODO: Might make since to put starting hands in the deck class for starting_hand in self._starting_hand_probs[p_i].keys(): starting_hand = list( starting_hand ) - # TODO: is this bad? + # TODO: Is this bad? if "p_reach" in locals(): del p_reach action_sequence: Dict[str, List[str]] = collections.defaultdict(list) @@ -406,9 +403,9 @@ def _update_hole_cards_bayes(self, offline_strategy: Dict[str, Dict[str, for i in range(n_actions_round): action = self._history[betting_stage][i] while action == 'skip': - i += 1 # action sequences don't end in skip + i += 1 # Action sequences don't end in skip action = self._history[betting_stage][i] - # TODO: maybe a method already exists for this? + # TODO: Maybe a method already exists for this? if betting_stage == "pre_flop": ph = (i + 2) % n_players else: @@ -444,16 +441,15 @@ def _update_hole_cards_bayes(self, offline_strategy: Dict[str, Dict[str, history=action_sequence, this_betting_stage=betting_stage, ) - # check to see if the strategy exists, + # Check to see if the strategy exists, # if not equal probability # TODO: is this overly hacky? # Problem with defaulting to 1 / 3, is that it # it doesn't work for calculations that # need to be made with the object's values - - try: # TODO: with or without keys + try: prob = offline_strategy[infoset][action] - # Normalizing since offline_stregy is not + # Normalizing unnormalized offline_stregy prob /= sum(offline_strategy[infoset]\ .values()) except KeyError: @@ -483,10 +479,9 @@ def _update_hole_cards_bayes(self, offline_strategy: Dict[str, Dict[str, history=action_sequence, this_betting_stage=betting_stage, ) - # TODO: Check this try: total_prob = offline_strategy[infoset][action] - # Normalizing since offline_stregy is not + # Normalizing unnormalized offline_stregy total_prob /= sum(offline_strategy[infoset]\ .values()) except KeyError: @@ -502,7 +497,6 @@ def _update_hole_cards_bayes(self, offline_strategy: Dict[str, Dict[str, def _get_starting_hand(self, player_idx: int) -> List[Card]: """Get starting hand based on probability of reach""" starting_hands = list(self._starting_hand_probs[player_idx].keys()) - # hacky for using tuples as keys starting_hands_idxs = list(range(len(starting_hands))) starting_hands_probs = list(self._starting_hand_probs[ player_idx diff --git a/research/test_methodology/RT.py b/research/test_methodology/RT.py index b79adbb4..2391d50e 100644 --- a/research/test_methodology/RT.py +++ b/research/test_methodology/RT.py @@ -3,24 +3,25 @@ import dill as pickle -from RT_cfr import train +from RT_cfr import rts from pluribus.poker.card import Card -from pluribus.games.short_deck.agent import Agent if __name__ == "__main__": + # We can set public cards or not # public_cards = [Card("ace", "spades"), Card("queen", "spades"), # Card("queen", "hearts")] public_cards: List[Card] = [] - # we load a (trained) strategy - agent = Agent(regret_dir='test_strategy/strategy_100.gz') + # Action sequence must be in old form (one list, includes skips) action_sequence = ["raise", "call", "call", "call", "call"] - agent_output, offline_strategy = train( + agent_output, offline_strategy = rts( 'test_strategy/unnormalized_output/offline_strategy_100.gz', 'test_strategy/strategy_100.gz', public_cards, action_sequence, 40, 6, 6, 3, 2, 6, 10 ) - with open("testing2.pkl", "wb") as file: - pickle.dump(agent_output, file) - import ipdb + save_path = "test_strategy/unnormalized_output/" + last_regret = {info_set: dict(strategy) for info_set, strategy in agent_output.regret.items()} + joblib.dump(offline_strategy, save_path + 'rts_output.gz', compress="gzip") + joblib.dump(last_regret, save_path + 'last_regret.gz', compress="gzip") + import ipdb; ipdb.set_trace() diff --git a/research/test_methodology/RT_cfr.py b/research/test_methodology/RT_cfr.py index a4bfd363..908fd3e2 100644 --- a/research/test_methodology/RT_cfr.py +++ b/research/test_methodology/RT_cfr.py @@ -1,11 +1,8 @@ from __future__ import annotations import logging -import sys import collections from typing import Dict -import copy -import json import joblib from tqdm import trange @@ -15,19 +12,9 @@ from pluribus.games.short_deck.state import ShortDeckPokerState, new_game from pluribus.games.short_deck.agent import Agent -sys.path.append("../blueprint_algo") - - - - -def to_dict(**kwargs) -> Dict[str, Any]: - """Hacky method to convert weird collections dicts to regular dicts.""" - return json.loads(json.dumps(copy.deepcopy(kwargs))) - def normalize_strategy(this_info_sets_regret: Dict[str, float]) -> Dict[str, float]: """Calculate the strategy based on the current information sets regret.""" - # TODO: Could we instanciate a state object from an info set? actions = this_info_sets_regret.keys() regret_sum = sum([max(regret, 0) for regret in this_info_sets_regret.values()]) if regret_sum > 0: @@ -46,43 +33,13 @@ def normalize_strategy(this_info_sets_regret: Dict[str, float]) -> Dict[str, flo return strategy -def update_strategy(agent: Agent, state: ShortDeckPokerState, ph_test_node: int): - """ - Update strategy for test node only - """ - ph = state.player_i - if ph == ph_test_node: - I = state.info_set - # calculate regret - sigma = calculate_strategy(agent.regret, I, state, offline_strategy, - unnormalized_strategy) - # choose an action based of sigma - try: - a = np.random.choice(list(sigma[I].keys()), 1, p=list(sigma[I].values()))[0] - except ValueError: - p = 1 / len(state.legal_actions) - probabilities = np.full(len(state.legal_actions), p) - a = np.random.choice(state.legal_actions, p=probabilities) - sigma[I] = {action: p for action in state.legal_actions} - # Increment the action counter. - agent.strategy[I][a] += 1 - return - else: - return - - def calculate_strategy( regret: Dict[str, Dict[str, float]], I: str, state: ShortDeckPokerState, -): +) -> Dict[str, Dict[str, float]]: """ - - :param regret: dictionary of regrets, I is key, then each action at I, with values being regret - :param sigma: dictionary of strategy updated by regret, iteration is key, then I is key, then each action with prob - :param I: - :param state: the game state - :return: doesn't return anything, just updates sigma + Calculate strategy based on regret """ sigma = collections.defaultdict(lambda: collections.defaultdict(lambda: 1 / 3)) rsum = sum([max(x, 0) for x in regret[I].values()]) @@ -96,12 +53,7 @@ def calculate_strategy( def cfr(agent: Agent, state: ShortDeckPokerState, i: int, t: int) -> float: """ - regular cfr algo - - :param state: the game state - :param i: player - :param t: iteration - :return: expected value for node for player i + CFR algo with the a temporary regret object for better strategy averaging """ ph = state.player_i @@ -109,24 +61,9 @@ def cfr(agent: Agent, state: ShortDeckPokerState, i: int, t: int) -> float: if state.is_terminal or player_not_in_hand: return state.payout[i] - # NOTE(fedden): The logic in Algorithm 1 in the supplementary material - # instructs the following lines of logic, but state class - # will already skip to the next in-hand player. - # elif p_i not in hand: - # cfr() - # NOTE(fedden): According to Algorithm 1 in the supplementary material, - # we would add in the following bit of logic. However we - # already have the game logic embedded in the state class, - # and this accounts for the chance samplings. In other words, - # it makes sure that chance actions such as dealing cards - # happen at the appropriate times. - # elif h is chance_node: - # sample action from strategy for h - # cfr() - elif ph == i: I = state.info_set - # calculate strategy + # Move regret over to temporary object and build off that if agent.tmp_regret[I] == {}: agent.tmp_regret[I] == agent.regret[I].copy() sigma = calculate_strategy(agent.tmp_regret, I, state) @@ -144,6 +81,7 @@ def cfr(agent: Agent, state: ShortDeckPokerState, i: int, t: int) -> float: return vo else: Iph = state.info_set + # Move regret over to a temporary object and build off that if agent.tmp_regret[Iph] == {}: agent.tmp_regret[Iph] == agent.regret[Iph].copy() sigma = calculate_strategy(agent.regret, Iph, state) @@ -162,7 +100,7 @@ def cfr(agent: Agent, state: ShortDeckPokerState, i: int, t: int) -> float: return cfr(agent, new_state, i, t) -def train( +def rts( offline_strategy_path: str, regret_path: str, public_cards: list, @@ -175,29 +113,26 @@ def train( update_threshold: int, dump_int: int, ): - """Train agent.""" + """RTS.""" # TODO: fix the seed utils.random.seed(36) - agent = Agent(regret_dir=regret_path) - + agent = Agent(regret_path=regret_path) + # Load unnormalized strategy to build off offline_strategy = joblib.load(offline_strategy_path) state: ShortDeckPokerState = new_game( 3, real_time_test=True, public_cards=public_cards ) + # Load current game state current_game_state: ShortDeckPokerState = state.load_game_state( offline_strategy, action_sequence ) + # We don't need the offline strategy for search.. del offline_strategy - # ph_test_node = current_game_state.player_i for t in trange(1, n_iterations + 1, desc="train iter"): print(t) - if t == 2: - logging.disable(logging.DEBUG) for i in range(n_players): # fixed position i - # Create a new state. + # Deal hole cards based on bayesian updating of hole card probs state: ShortDeckPokerState = current_game_state.deal_bayes() -# if t % update_interval == 0 and t > update_threshold: -# update_strategy(agent, state, ph_test_node) cfr(agent, state, i, t) if t < lcfr_threshold & t % discount_interval == 0: d = (t / discount_interval) / ((t / discount_interval) + 1) @@ -206,7 +141,7 @@ def train( agent.tmp_regret[I][a] *= d offline_strategy = joblib.load(offline_strategy_path) - # Adding the regret back to the regret dict + # Adding the regret back to the regret dict, we'll build off for next RTS for I in agent.tmp_regret.keys(): if agent.tmp_regret != {}: agent.regret[I] = agent.tmp_regret[I].copy() @@ -215,6 +150,7 @@ def train( for info_set, this_info_sets_regret in sorted(agent.tmp_regret.items()): # If this_info_sets_regret == {}, we do nothing strategy = normalize_strategy(this_info_sets_regret) + # Check if info_set exists.. if info_set not in offline_strategy: offline_strategy[info_set] = {a: 0 for a in strategy.keys()} for action, probability in strategy.items(): diff --git a/research/test_methodology/average_unnormalized_strategy.py b/research/test_methodology/average_unnormalized_strategy.py index 08cf665f..e35965ad 100644 --- a/research/test_methodology/average_unnormalized_strategy.py +++ b/research/test_methodology/average_unnormalized_strategy.py @@ -11,7 +11,6 @@ def calculate_strategy(this_info_sets_regret: Dict[str, float]) -> Dict[str, float]: """Calculate the strategy based on the current information sets regret.""" - # TODO: Could we instanciate a state object from an info set? actions = this_info_sets_regret.keys() regret_sum = sum([max(regret, 0) for regret in this_info_sets_regret.values()]) if regret_sum > 0: From 0cd1bbaae81997aaf536f32f4e2541c87d636acd Mon Sep 17 00:00:00 2001 From: big-c-note Date: Sun, 24 May 2020 00:58:28 -0400 Subject: [PATCH 27/42] beginnings of a test script --- research/test_methodology/test_RT.py | 84 ++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 research/test_methodology/test_RT.py diff --git a/research/test_methodology/test_RT.py b/research/test_methodology/test_RT.py new file mode 100644 index 00000000..69277f0a --- /dev/null +++ b/research/test_methodology/test_RT.py @@ -0,0 +1,84 @@ +from typing import List, Dict, DefaultDict +import joblib +import collections + +from tqdm import trange +import numpy as np +from scipy import stats + +from pluribus.games.short_deck.state import ShortDeckPokerState, new_game +from pluribus.poker.card import Card + + +def _calculate_strategy( + state: ShortDeckPokerState, + I: str, + strategy: DefaultDict[str, DefaultDict[str, float]] +) -> str: + sigma = collections.defaultdict(lambda: collections.defaultdict(lambda: 1 / 3)) + try: + # If strategy is empty, go to other block + if sigma[I] == {}: + raise KeyError + sigma[I] = strategy[I].copy() + norm = sum(sigma[I].values()) + for a in sigma[I].keys(): + sigma[I][a] /= norm + a = np.random.choice( + list(sigma[I].keys()), 1, p=list(sigma[I].values()), + )[0] + except KeyError: + p = 1 / len(state.legal_actions) + probabilities = np.full(len(state.legal_actions), p) + a = np.random.choice(state.legal_actions, p=probabilities) + sigma[I] = {action: p for action in state.legal_actions} + return a + + +# Load unnormalized strategy before rts +offline_path = 'test_strategy/unnormalized_output/offline_strategy_100.gz' +offline_strategy = joblib.load(offline_path) +# Load unnormalized strategy with rts +offline_rts_path = 'test_strategy/unnormalized_output/rts_output.gz' +offline_strategy_rts = joblib.load(offline_rts_path) + +public_cards: List[Card] = [] +action_sequence = ["raise", "call", "call", "call", "call"] +# Loading game state we used RTS on +state: ShortDeckPokerState = new_game( + 3, real_time_test=True, public_cards=public_cards +) +# Load current game state, either strategy should be identical for this +current_game_state: ShortDeckPokerState = state.load_game_state( + offline_strategy, action_sequence +) +n_inner_iters = 100 +n_outter_iters = 30 +EVs = np.array([]) +# We don't need the offline strategy for search.. +for _ in trange(1, n_outter_iters): + EV = np.array([]) # Expected value for player 0 (hero) + for t in trange(1, n_inner_iters + 1, desc="train iter"): + for p_i in range(3): + # Deal hole cards based on bayesian updating of hole card probs + state: ShortDeckPokerState = current_game_state.deal_bayes() + while True: + player_not_in_hand = not state.players[p_i].is_active + if state.is_terminal or player_not_in_hand: + EV = np.append(EV, state.payout[p_i]) + break + if state.player_i == p_i: + random_action: str = _calculate_strategy(state, state.info_set, + offline_strategy_rts) + else: + random_action: str = _calculate_strategy(state, state.info_set, + offline_strategy) + state = state.apply_action(random_action) + EVs = np.append(EVs, EV.mean()) +print(f"Average EV after 30 sets of 100 games: {EVs.mean()}") +t_stat = (EVs.mean() - 0) / (EVs.std() / np.sqrt(n_outter_iters)) +print(f"T statistic = {t_stat}") +p_val = stats.t.sf(np.abs(t_stat), n_outter_iters - 1) +print(f"P-Value: {p_val}") +import ipdb; +ipdb.set_trace() From f2854421a5aef982d30eadc44b90de45cf59455d Mon Sep 17 00:00:00 2001 From: big-c-note Date: Mon, 25 May 2020 10:11:09 -0400 Subject: [PATCH 28/42] updating offline_strategy on each dump int, wrapping test method into function, fix broken test --- pluribus/games/short_deck/agent.py | 12 +-- research/test_methodology/RT.py | 12 +-- research/test_methodology/RT_cfr.py | 40 +++++--- research/test_methodology/bot_test.py | 139 ++++++++++++++++++++++++++ research/test_methodology/test_RT.py | 84 ---------------- 5 files changed, 174 insertions(+), 113 deletions(-) create mode 100644 research/test_methodology/bot_test.py delete mode 100644 research/test_methodology/test_RT.py diff --git a/pluribus/games/short_deck/agent.py b/pluribus/games/short_deck/agent.py index c55ab503..92eee131 100644 --- a/pluribus/games/short_deck/agent.py +++ b/pluribus/games/short_deck/agent.py @@ -21,9 +21,9 @@ def __init__(self, regret_path=None): lambda: collections.defaultdict(lambda: 0) ) - def reset_new_regret(self): - """Remove regret from temporary storage""" - del self.tmp_regret - self.tmp_regret = collections.defaultdict( - lambda: collections.defaultdict(lambda: 0) - ) + def reset_new_regret(self): + """Remove regret from temporary storage""" + del self.tmp_regret + self.tmp_regret = collections.defaultdict( + lambda: collections.defaultdict(lambda: 0) + ) diff --git a/research/test_methodology/RT.py b/research/test_methodology/RT.py index 2391d50e..20d7388b 100644 --- a/research/test_methodology/RT.py +++ b/research/test_methodology/RT.py @@ -1,23 +1,21 @@ from typing import List import joblib -import dill as pickle - from RT_cfr import rts from pluribus.poker.card import Card if __name__ == "__main__": # We can set public cards or not - # public_cards = [Card("ace", "spades"), Card("queen", "spades"), - # Card("queen", "hearts")] - public_cards: List[Card] = [] + public_cards = [Card("king", "spades"), Card("queen", "clubs"), + Card("jack", "hearts")] + # public_cards: List[Card] = [] # Action sequence must be in old form (one list, includes skips) - action_sequence = ["raise", "call", "call", "call", "call"] + action_sequence = ["raise", "call", "call", "call"] agent_output, offline_strategy = rts( 'test_strategy/unnormalized_output/offline_strategy_100.gz', 'test_strategy/strategy_100.gz', public_cards, action_sequence, - 40, 6, 6, 3, 2, 6, 10 + 140, 6, 6, 3, 2, 6, 10 ) save_path = "test_strategy/unnormalized_output/" last_regret = {info_set: dict(strategy) for info_set, strategy in agent_output.regret.items()} diff --git a/research/test_methodology/RT_cfr.py b/research/test_methodology/RT_cfr.py index 908fd3e2..3a43bd2a 100644 --- a/research/test_methodology/RT_cfr.py +++ b/research/test_methodology/RT_cfr.py @@ -127,7 +127,7 @@ def rts( offline_strategy, action_sequence ) # We don't need the offline strategy for search.. - del offline_strategy + # del offline_strategy for t in trange(1, n_iterations + 1, desc="train iter"): print(t) for i in range(n_players): # fixed position i @@ -139,20 +139,28 @@ def rts( for I in agent.tmp_regret.keys(): for a in agent.tmp_regret[I].keys(): agent.tmp_regret[I][a] *= d + # Add the unnormalized strategy into the original + # Right now assumes dump_int is a multiple of n_iterations + if t % dump_int == 0: + # offline_strategy = joblib.load(offline_strategy_path) + # Adding the regret back to the regret dict, we'll build off for next RTS + for I in agent.tmp_regret.keys(): + if agent.tmp_regret != {}: + agent.regret[I] = agent.tmp_regret[I].copy() + for info_set, this_info_sets_regret in sorted(agent.tmp_regret.items()): + # If this_info_sets_regret == {}, we do nothing + strategy = normalize_strategy(this_info_sets_regret) + # Check if info_set exists.. + no_info_set = info_set not in offline_strategy + if no_info_set or offline_strategy[info_set] == {}: + offline_strategy[info_set] = {a: 0 for a in strategy.keys()} + for action, probability in strategy.items(): + try: + offline_strategy[info_set][action] += probability + except: + import ipdb; + ipdb.set_trace() + agent.reset_new_regret() + - offline_strategy = joblib.load(offline_strategy_path) - # Adding the regret back to the regret dict, we'll build off for next RTS - for I in agent.tmp_regret.keys(): - if agent.tmp_regret != {}: - agent.regret[I] = agent.tmp_regret[I].copy() - - # Add the unnormalized strategy into the original - for info_set, this_info_sets_regret in sorted(agent.tmp_regret.items()): - # If this_info_sets_regret == {}, we do nothing - strategy = normalize_strategy(this_info_sets_regret) - # Check if info_set exists.. - if info_set not in offline_strategy: - offline_strategy[info_set] = {a: 0 for a in strategy.keys()} - for action, probability in strategy.items(): - offline_strategy[info_set][action] += t / dump_int * probability return agent, offline_strategy diff --git a/research/test_methodology/bot_test.py b/research/test_methodology/bot_test.py new file mode 100644 index 00000000..376f7a9a --- /dev/null +++ b/research/test_methodology/bot_test.py @@ -0,0 +1,139 @@ +from typing import List, Dict, DefaultDict +from pathlib import Path +import joblib +import collections + +from tqdm import trange +import yaml +import datetime +import numpy as np +from scipy import stats + +from pluribus.games.short_deck.state import ShortDeckPokerState, new_game +from pluribus.poker.card import Card + + +def _calculate_strategy( + state: ShortDeckPokerState, + I: str, + strategy: DefaultDict[str, DefaultDict[str, float]] +) -> str: + sigma = collections.defaultdict(lambda: collections.defaultdict(lambda: 1 / 3)) + import ipdb; + ipdb.set_trace() + + try: + # If strategy is empty, go to other block + if sigma[I] == {}: + raise KeyError + sigma[I] = strategy[I].copy() + norm = sum(sigma[I].values()) + for a in sigma[I].keys(): + sigma[I][a] /= norm + a = np.random.choice( + list(sigma[I].keys()), 1, p=list(sigma[I].values()), + )[0] + except KeyError: + p = 1 / len(state.legal_actions) + probabilities = np.full(len(state.legal_actions), p) + a = np.random.choice(state.legal_actions, p=probabilities) + sigma[I] = {action: p for action in state.legal_actions} + return a + + +def _create_dir() -> Path: + """Create and get a unique dir path to save to using a timestamp.""" + time = str(datetime.datetime.now()) + for char in ":- .": + time = time.replace(char, "_") + path: Path = Path(f"./results_{time}") + path.mkdir(parents=True, exist_ok=True) + return path + + +def agent_test( + hero_strategy_path: str, + opponent_strategy_path: str, + real_time_est: bool = False, + action_sequence: List[str] = None, + public_cards: List[Card] = [], + n_outter_iters: int = 30, + n_inner_iters: int = 100, + n_players: int = 3, +): + config: Dict[str, int] = {**locals()} + save_path: Path = _create_dir() + with open(save_path / "config.yaml", "w") as steam: + yaml.dump(config, steam) + + # Load unnormalized strategy for hero + hero_strategy = joblib.load(hero_strategy_path) + # Load unnormalized strategy for opponents + opponent_strategy = joblib.load(opponent_strategy_path) + + # Loading game state we used RTS on + if real_time_est: + state: ShortDeckPokerState = new_game( + n_players, real_time_test=real_time_est, public_cards=public_cards + ) + current_game_state: ShortDeckPokerState = state.load_game_state( + opponent_strategy, action_sequence + ) + + # TODO: Right now, this can only be used for loading states if the two strategies + # are averaged. Even averaging strategies is risky. Loading a game state + # should be used with caution. It will work only if the probability of reach + # is identical across strategies. Use the average strategy. + + # Load current game state, either strategy should be identical for this + EVs = np.array([]) + for _ in trange(1, n_outter_iters): + EV = np.array([]) # Expected value for player 0 (hero) + for t in trange(1, n_inner_iters + 1, desc="train iter"): + for p_i in range(n_players): + if real_time_est: + # Deal hole cards based on bayesian updating of hole card probs + state: ShortDeckPokerState = current_game_state.deal_bayes() + else: + state: ShortDeckPokerState = new_game(n_players) + while True: + player_not_in_hand = not state.players[p_i].is_active + if state.is_terminal or player_not_in_hand: + EV = np.append(EV, state.payout[p_i]) + break + if state.player_i == p_i: + random_action: str = _calculate_strategy( + state, + state.info_set, + hero_strategy, + ) + else: + random_action: str = _calculate_strategy( + state, + state.info_set, + opponent_strategy, + ) + state = state.apply_action(random_action) + EVs = np.append(EVs, EV.mean()) + t_stat = (EVs.mean() - 0) / (EVs.std() / np.sqrt(n_outter_iters)) + p_val = stats.t.sf(np.abs(t_stat), n_outter_iters - 1) + results_dict = { + 'Expected Value': float(EVs.mean()), + 'T Statistic': float(t_stat), + 'P Value': float(p_val) + } + with open(save_path / 'results.yaml', "w") as stream: + yaml.safe_dump(results_dict, stream=stream, default_flow_style=False) + + +if __name__ == "__main__": + strat_path = "test_strategy/unnormalized_output/" + agent_test( + hero_strategy_path=strat_path + "rts_output.gz", + opponent_strategy_path=strat_path + "offline_strategy_100.gz", + real_time_est=True, + public_cards=[Card("ace", "spades"), Card("queen", "spades"), + Card("queen", "hearts")], + action_sequence=["raise", "call", "call", "call", "call"], + n_inner_iters=100 + ) diff --git a/research/test_methodology/test_RT.py b/research/test_methodology/test_RT.py deleted file mode 100644 index 69277f0a..00000000 --- a/research/test_methodology/test_RT.py +++ /dev/null @@ -1,84 +0,0 @@ -from typing import List, Dict, DefaultDict -import joblib -import collections - -from tqdm import trange -import numpy as np -from scipy import stats - -from pluribus.games.short_deck.state import ShortDeckPokerState, new_game -from pluribus.poker.card import Card - - -def _calculate_strategy( - state: ShortDeckPokerState, - I: str, - strategy: DefaultDict[str, DefaultDict[str, float]] -) -> str: - sigma = collections.defaultdict(lambda: collections.defaultdict(lambda: 1 / 3)) - try: - # If strategy is empty, go to other block - if sigma[I] == {}: - raise KeyError - sigma[I] = strategy[I].copy() - norm = sum(sigma[I].values()) - for a in sigma[I].keys(): - sigma[I][a] /= norm - a = np.random.choice( - list(sigma[I].keys()), 1, p=list(sigma[I].values()), - )[0] - except KeyError: - p = 1 / len(state.legal_actions) - probabilities = np.full(len(state.legal_actions), p) - a = np.random.choice(state.legal_actions, p=probabilities) - sigma[I] = {action: p for action in state.legal_actions} - return a - - -# Load unnormalized strategy before rts -offline_path = 'test_strategy/unnormalized_output/offline_strategy_100.gz' -offline_strategy = joblib.load(offline_path) -# Load unnormalized strategy with rts -offline_rts_path = 'test_strategy/unnormalized_output/rts_output.gz' -offline_strategy_rts = joblib.load(offline_rts_path) - -public_cards: List[Card] = [] -action_sequence = ["raise", "call", "call", "call", "call"] -# Loading game state we used RTS on -state: ShortDeckPokerState = new_game( - 3, real_time_test=True, public_cards=public_cards -) -# Load current game state, either strategy should be identical for this -current_game_state: ShortDeckPokerState = state.load_game_state( - offline_strategy, action_sequence -) -n_inner_iters = 100 -n_outter_iters = 30 -EVs = np.array([]) -# We don't need the offline strategy for search.. -for _ in trange(1, n_outter_iters): - EV = np.array([]) # Expected value for player 0 (hero) - for t in trange(1, n_inner_iters + 1, desc="train iter"): - for p_i in range(3): - # Deal hole cards based on bayesian updating of hole card probs - state: ShortDeckPokerState = current_game_state.deal_bayes() - while True: - player_not_in_hand = not state.players[p_i].is_active - if state.is_terminal or player_not_in_hand: - EV = np.append(EV, state.payout[p_i]) - break - if state.player_i == p_i: - random_action: str = _calculate_strategy(state, state.info_set, - offline_strategy_rts) - else: - random_action: str = _calculate_strategy(state, state.info_set, - offline_strategy) - state = state.apply_action(random_action) - EVs = np.append(EVs, EV.mean()) -print(f"Average EV after 30 sets of 100 games: {EVs.mean()}") -t_stat = (EVs.mean() - 0) / (EVs.std() / np.sqrt(n_outter_iters)) -print(f"T statistic = {t_stat}") -p_val = stats.t.sf(np.abs(t_stat), n_outter_iters - 1) -print(f"P-Value: {p_val}") -import ipdb; -ipdb.set_trace() From 203fb222a5c4a59b29fa7f76a8bd1c2567598320 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Mon, 25 May 2020 10:27:12 -0400 Subject: [PATCH 29/42] sample out put of state class producing same results as develop branch after changes --- research/blueprint_algo/after_RT.txt | 307 ++++++++++++++++++++++++++ research/blueprint_algo/before_RT.txt | 307 ++++++++++++++++++++++++++ 2 files changed, 614 insertions(+) create mode 100644 research/blueprint_algo/after_RT.txt create mode 100644 research/blueprint_algo/before_RT.txt diff --git a/research/blueprint_algo/after_RT.txt b/research/blueprint_algo/after_RT.txt new file mode 100644 index 00000000..b669b4a6 --- /dev/null +++ b/research/blueprint_algo/after_RT.txt @@ -0,0 +1,307 @@ +DEBUG:pluribus.poker.engine:Assigned blinds to players [, ] +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 2 +DEBUG:root:P(h) Updating Regret? False +DEBUG:root:Betting Round pre_flop +DEBUG:root:Community Cards [] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): {"cards_cluster":18,"history":[]} +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x1133139d8>, {}) +DEBUG:root:Current regret: defaultdict(... at 0x1133139d8>, {}) +DEBUG:root:Calculated Strategy for {"cards_cluster":18,"history":[]}: defaultdict(... at 0x113316378>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) +DEBUG:root:ACTION SAMPLED: ph 2 ACTION: fold +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 0 +DEBUG:root:P(h) Updating Regret? True +DEBUG:root:Betting Round pre_flop +DEBUG:root:Community Cards [] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): {"cards_cluster":24,"history":[{"pre_flop":["fold"]}]} +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x113316268>, {}) +DEBUG:root:Current regret: defaultdict(... at 0x113316268>, {}) +DEBUG:root:Calculated Strategy for {"cards_cluster":24,"history":[{"pre_flop":["fold"]}]}: defaultdict(... at 0x1133161e0>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) +DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: fold +DEBUG:pluribus.poker.engine:f"Rank #3776 pair +DEBUG:pluribus.poker.engine:Winnings computation complete. Players: +DEBUG:pluribus.poker.engine: +DEBUG:pluribus.poker.engine: +DEBUG:pluribus.poker.engine: +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 2 +DEBUG:root:P(h) Updating Regret? False +DEBUG:root:Betting Round terminal +DEBUG:root:Community Cards [, , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): default info set, please ensure you load it correctly +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:Got EV for fold: -50 +DEBUG:root:Added to Node EV for ACTION: fold INFOSET: {"cards_cluster":24,"history":[{"pre_flop":["fold"]}]} + STRATEGY: 0.3333333333333333: -16.666666666666664 +DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: call +DEBUG:pluribus.games.short_deck.state:calling +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 1 +DEBUG:root:P(h) Updating Regret? False +DEBUG:root:Betting Round pre_flop +DEBUG:root:Community Cards [] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): {"cards_cluster":22,"history":[{"pre_flop":["fold","call"]}]} +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x113316158>, {}) +DEBUG:root:Current regret: defaultdict(... at 0x113316158>, {}) +DEBUG:root:Calculated Strategy for {"cards_cluster":22,"history":[{"pre_flop":["fold","call"]}]}: defaultdict(... at 0x1133160d0>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) +DEBUG:root:ACTION SAMPLED: ph 1 ACTION: call +DEBUG:pluribus.games.short_deck.state:calling +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 0 +DEBUG:root:P(h) Updating Regret? True +DEBUG:root:Betting Round flop +DEBUG:root:Community Cards [, , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): {"cards_cluster":1,"history":[{"pre_flop":["fold","call","call"]}]} +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x113316950>, {}) +DEBUG:root:Current regret: defaultdict(... at 0x113316950>, {}) +DEBUG:root:Calculated Strategy for {"cards_cluster":1,"history":[{"pre_flop":["fold","call","call"]}]}: defaultdict(... at 0x1133169d8>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) +DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: fold +DEBUG:pluribus.poker.engine:f"Rank #3986 pair +DEBUG:pluribus.poker.engine:Winnings computation complete. Players: +DEBUG:pluribus.poker.engine: +DEBUG:pluribus.poker.engine: +DEBUG:pluribus.poker.engine: +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 1 +DEBUG:root:P(h) Updating Regret? False +DEBUG:root:Betting Round terminal +DEBUG:root:Community Cards [, , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): default info set, please ensure you load it correctly +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:Got EV for fold: -100 +DEBUG:root:Added to Node EV for ACTION: fold INFOSET: {"cards_cluster":1,"history":[{"pre_flop":["fold","call","call"]}]} + STRATEGY: 0.3333333333333333: -33.33333333333333 +DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: call +DEBUG:pluribus.games.short_deck.state:calling +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 1 +DEBUG:root:P(h) Updating Regret? False +DEBUG:root:Betting Round flop +DEBUG:root:Community Cards [, , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): {"cards_cluster":22,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call"]}]} +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x113316ae8>, {}) +DEBUG:root:Current regret: defaultdict(... at 0x113316ae8>, {}) +DEBUG:root:Calculated Strategy for {"cards_cluster":22,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call"]}]}: defaultdict(... at 0x113316bf8>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) +DEBUG:root:ACTION SAMPLED: ph 1 ACTION: raise +DEBUG:pluribus.games.short_deck.state:betting 100 n chips +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 0 +DEBUG:root:P(h) Updating Regret? True +DEBUG:root:Betting Round flop +DEBUG:root:Community Cards [, , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): {"cards_cluster":1,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise"]}]} +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x113316d08>, {}) +DEBUG:root:Current regret: defaultdict(... at 0x113316d08>, {}) +DEBUG:root:Calculated Strategy for {"cards_cluster":1,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise"]}]}: defaultdict(... at 0x113316d90>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) +DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: fold +DEBUG:pluribus.poker.engine:f"Rank #2721 two pair +DEBUG:pluribus.poker.engine:Winnings computation complete. Players: +DEBUG:pluribus.poker.engine: +DEBUG:pluribus.poker.engine: +DEBUG:pluribus.poker.engine: +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 1 +DEBUG:root:P(h) Updating Regret? False +DEBUG:root:Betting Round terminal +DEBUG:root:Community Cards [, , , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): default info set, please ensure you load it correctly +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:Got EV for fold: -100 +DEBUG:root:Added to Node EV for ACTION: fold INFOSET: {"cards_cluster":1,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise"]}]} + STRATEGY: 0.3333333333333333: -33.33333333333333 +DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: call +DEBUG:pluribus.games.short_deck.state:calling +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 0 +DEBUG:root:P(h) Updating Regret? True +DEBUG:root:Betting Round turn +DEBUG:root:Community Cards [, , , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): {"cards_cluster":24,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]}]} +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x113316e18>, {}) +DEBUG:root:Current regret: defaultdict(... at 0x113316e18>, {}) +DEBUG:root:Calculated Strategy for {"cards_cluster":24,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]}]}: defaultdict(... at 0x113316f28>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) +DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: fold +DEBUG:pluribus.poker.engine:f"Rank #2721 two pair +DEBUG:pluribus.poker.engine:Winnings computation complete. Players: +DEBUG:pluribus.poker.engine: +DEBUG:pluribus.poker.engine: +DEBUG:pluribus.poker.engine: +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 1 +DEBUG:root:P(h) Updating Regret? False +DEBUG:root:Betting Round terminal +DEBUG:root:Community Cards [, , , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): default info set, please ensure you load it correctly +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:Got EV for fold: -200 +DEBUG:root:Added to Node EV for ACTION: fold INFOSET: {"cards_cluster":24,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]}]} + STRATEGY: 0.3333333333333333: -66.66666666666666 +DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: call +DEBUG:pluribus.games.short_deck.state:calling +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 1 +DEBUG:root:P(h) Updating Regret? False +DEBUG:root:Betting Round turn +DEBUG:root:Community Cards [, , , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): {"cards_cluster":23,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call"]}]} +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x113d01048>, {}) +DEBUG:root:Current regret: defaultdict(... at 0x113d01048>, {}) +DEBUG:root:Calculated Strategy for {"cards_cluster":23,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call"]}]}: defaultdict(... at 0x113d01158>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) +DEBUG:root:ACTION SAMPLED: ph 1 ACTION: raise +DEBUG:pluribus.games.short_deck.state:betting 200 n chips +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 0 +DEBUG:root:P(h) Updating Regret? True +DEBUG:root:Betting Round turn +DEBUG:root:Community Cards [, , , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): {"cards_cluster":24,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call","raise"]}]} +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x113d01268>, {}) +DEBUG:root:Current regret: defaultdict(... at 0x113d01268>, {}) +DEBUG:root:Calculated Strategy for {"cards_cluster":24,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call","raise"]}]}: defaultdict(... at 0x113d012f0>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) +DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: fold +DEBUG:pluribus.poker.engine:f"Rank #205 full house +DEBUG:pluribus.poker.engine:Winnings computation complete. Players: +DEBUG:pluribus.poker.engine: +DEBUG:pluribus.poker.engine: +DEBUG:pluribus.poker.engine: +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 1 +DEBUG:root:P(h) Updating Regret? False +DEBUG:root:Betting Round terminal +DEBUG:root:Community Cards [, , , , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): default info set, please ensure you load it correctly +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:Got EV for fold: -200 +DEBUG:root:Added to Node EV for ACTION: fold INFOSET: {"cards_cluster":24,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call","raise"]}]} + STRATEGY: 0.3333333333333333: -66.66666666666666 +DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: call +DEBUG:pluribus.games.short_deck.state:calling +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 0 +DEBUG:root:P(h) Updating Regret? True +DEBUG:root:Betting Round river +DEBUG:root:Community Cards [, , , , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): {"cards_cluster":10,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call","raise","skip","call"]}]} +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x113d01378>, {}) +DEBUG:root:Current regret: defaultdict(... at 0x113d01378>, {}) +DEBUG:root:Calculated Strategy for {"cards_cluster":10,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call","raise","skip","call"]}]}: defaultdict(... at 0x113d01488>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) +DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: fold +DEBUG:pluribus.poker.engine:f"Rank #1600 straight +DEBUG:pluribus.poker.engine:Winnings computation complete. Players: +DEBUG:pluribus.poker.engine: +DEBUG:pluribus.poker.engine: +DEBUG:pluribus.poker.engine: +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 1 +DEBUG:root:P(h) Updating Regret? False +DEBUG:root:Betting Round terminal +DEBUG:root:Community Cards [, , , , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): default info set, please ensure you load it correctly +DEBUG:root:Betting Action Correct?: [, , ] diff --git a/research/blueprint_algo/before_RT.txt b/research/blueprint_algo/before_RT.txt new file mode 100644 index 00000000..0ec3e222 --- /dev/null +++ b/research/blueprint_algo/before_RT.txt @@ -0,0 +1,307 @@ +DEBUG:pluribus.poker.engine:Assigned blinds to players [, ] +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 2 +DEBUG:root:P(h) Updating Regret? False +DEBUG:root:Betting Round pre_flop +DEBUG:root:Community Cards [] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): {"cards_cluster":18,"history":[]} +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x10d79ed90>, {}) +DEBUG:root:Current regret: defaultdict(... at 0x10d79ed90>, {}) +DEBUG:root:Calculated Strategy for {"cards_cluster":18,"history":[]}: defaultdict(... at 0x10d79ed08>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) +DEBUG:root:ACTION SAMPLED: ph 2 ACTION: fold +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 0 +DEBUG:root:P(h) Updating Regret? True +DEBUG:root:Betting Round pre_flop +DEBUG:root:Community Cards [] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): {"cards_cluster":24,"history":[{"pre_flop":["fold"]}]} +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x10d79ebf8>, {}) +DEBUG:root:Current regret: defaultdict(... at 0x10d79ebf8>, {}) +DEBUG:root:Calculated Strategy for {"cards_cluster":24,"history":[{"pre_flop":["fold"]}]}: defaultdict(... at 0x10d79eb70>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) +DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: fold +DEBUG:pluribus.poker.engine:f"Rank #3776 pair +DEBUG:pluribus.poker.engine:Winnings computation complete. Players: +DEBUG:pluribus.poker.engine: +DEBUG:pluribus.poker.engine: +DEBUG:pluribus.poker.engine: +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 2 +DEBUG:root:P(h) Updating Regret? False +DEBUG:root:Betting Round terminal +DEBUG:root:Community Cards [, , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): default info set, please ensure you load it correctly +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:Got EV for fold: -50 +DEBUG:root:Added to Node EV for ACTION: fold INFOSET: {"cards_cluster":24,"history":[{"pre_flop":["fold"]}]} + STRATEGY: 0.3333333333333333: -16.666666666666664 +DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: call +DEBUG:pluribus.games.short_deck.state:calling +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 1 +DEBUG:root:P(h) Updating Regret? False +DEBUG:root:Betting Round pre_flop +DEBUG:root:Community Cards [] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): {"cards_cluster":22,"history":[{"pre_flop":["fold","call"]}]} +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x10d79eae8>, {}) +DEBUG:root:Current regret: defaultdict(... at 0x10d79eae8>, {}) +DEBUG:root:Calculated Strategy for {"cards_cluster":22,"history":[{"pre_flop":["fold","call"]}]}: defaultdict(... at 0x10d79e9d8>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) +DEBUG:root:ACTION SAMPLED: ph 1 ACTION: call +DEBUG:pluribus.games.short_deck.state:calling +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 0 +DEBUG:root:P(h) Updating Regret? True +DEBUG:root:Betting Round flop +DEBUG:root:Community Cards [, , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): {"cards_cluster":14,"history":[{"pre_flop":["fold","call","call"]}]} +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x10d79e950>, {}) +DEBUG:root:Current regret: defaultdict(... at 0x10d79e950>, {}) +DEBUG:root:Calculated Strategy for {"cards_cluster":14,"history":[{"pre_flop":["fold","call","call"]}]}: defaultdict(... at 0x10d7ae268>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) +DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: fold +DEBUG:pluribus.poker.engine:f"Rank #3986 pair +DEBUG:pluribus.poker.engine:Winnings computation complete. Players: +DEBUG:pluribus.poker.engine: +DEBUG:pluribus.poker.engine: +DEBUG:pluribus.poker.engine: +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 1 +DEBUG:root:P(h) Updating Regret? False +DEBUG:root:Betting Round terminal +DEBUG:root:Community Cards [, , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): default info set, please ensure you load it correctly +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:Got EV for fold: -100 +DEBUG:root:Added to Node EV for ACTION: fold INFOSET: {"cards_cluster":14,"history":[{"pre_flop":["fold","call","call"]}]} + STRATEGY: 0.3333333333333333: -33.33333333333333 +DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: call +DEBUG:pluribus.games.short_deck.state:calling +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 1 +DEBUG:root:P(h) Updating Regret? False +DEBUG:root:Betting Round flop +DEBUG:root:Community Cards [, , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): {"cards_cluster":22,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call"]}]} +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x10d7ae378>, {}) +DEBUG:root:Current regret: defaultdict(... at 0x10d7ae378>, {}) +DEBUG:root:Calculated Strategy for {"cards_cluster":22,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call"]}]}: defaultdict(... at 0x10d7ae488>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) +DEBUG:root:ACTION SAMPLED: ph 1 ACTION: raise +DEBUG:pluribus.games.short_deck.state:betting 100 n chips +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 0 +DEBUG:root:P(h) Updating Regret? True +DEBUG:root:Betting Round flop +DEBUG:root:Community Cards [, , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): {"cards_cluster":14,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise"]}]} +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x10d7ae598>, {}) +DEBUG:root:Current regret: defaultdict(... at 0x10d7ae598>, {}) +DEBUG:root:Calculated Strategy for {"cards_cluster":14,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise"]}]}: defaultdict(... at 0x10d7ae620>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) +DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: fold +DEBUG:pluribus.poker.engine:f"Rank #2721 two pair +DEBUG:pluribus.poker.engine:Winnings computation complete. Players: +DEBUG:pluribus.poker.engine: +DEBUG:pluribus.poker.engine: +DEBUG:pluribus.poker.engine: +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 1 +DEBUG:root:P(h) Updating Regret? False +DEBUG:root:Betting Round terminal +DEBUG:root:Community Cards [, , , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): default info set, please ensure you load it correctly +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:Got EV for fold: -100 +DEBUG:root:Added to Node EV for ACTION: fold INFOSET: {"cards_cluster":14,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise"]}]} + STRATEGY: 0.3333333333333333: -33.33333333333333 +DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: call +DEBUG:pluribus.games.short_deck.state:calling +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 0 +DEBUG:root:P(h) Updating Regret? True +DEBUG:root:Betting Round turn +DEBUG:root:Community Cards [, , , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): {"cards_cluster":25,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]}]} +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x10d7ae6a8>, {}) +DEBUG:root:Current regret: defaultdict(... at 0x10d7ae6a8>, {}) +DEBUG:root:Calculated Strategy for {"cards_cluster":25,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]}]}: defaultdict(... at 0x10d7ae7b8>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) +DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: fold +DEBUG:pluribus.poker.engine:f"Rank #2721 two pair +DEBUG:pluribus.poker.engine:Winnings computation complete. Players: +DEBUG:pluribus.poker.engine: +DEBUG:pluribus.poker.engine: +DEBUG:pluribus.poker.engine: +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 1 +DEBUG:root:P(h) Updating Regret? False +DEBUG:root:Betting Round terminal +DEBUG:root:Community Cards [, , , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): default info set, please ensure you load it correctly +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:Got EV for fold: -200 +DEBUG:root:Added to Node EV for ACTION: fold INFOSET: {"cards_cluster":25,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]}]} + STRATEGY: 0.3333333333333333: -66.66666666666666 +DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: call +DEBUG:pluribus.games.short_deck.state:calling +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 1 +DEBUG:root:P(h) Updating Regret? False +DEBUG:root:Betting Round turn +DEBUG:root:Community Cards [, , , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): {"cards_cluster":13,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call"]}]} +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x10d7ae840>, {}) +DEBUG:root:Current regret: defaultdict(... at 0x10d7ae840>, {}) +DEBUG:root:Calculated Strategy for {"cards_cluster":13,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call"]}]}: defaultdict(... at 0x10d7ae950>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) +DEBUG:root:ACTION SAMPLED: ph 1 ACTION: raise +DEBUG:pluribus.games.short_deck.state:betting 200 n chips +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 0 +DEBUG:root:P(h) Updating Regret? True +DEBUG:root:Betting Round turn +DEBUG:root:Community Cards [, , , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): {"cards_cluster":25,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call","raise"]}]} +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x10d7aea60>, {}) +DEBUG:root:Current regret: defaultdict(... at 0x10d7aea60>, {}) +DEBUG:root:Calculated Strategy for {"cards_cluster":25,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call","raise"]}]}: defaultdict(... at 0x10d7aeae8>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) +DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: fold +DEBUG:pluribus.poker.engine:f"Rank #205 full house +DEBUG:pluribus.poker.engine:Winnings computation complete. Players: +DEBUG:pluribus.poker.engine: +DEBUG:pluribus.poker.engine: +DEBUG:pluribus.poker.engine: +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 1 +DEBUG:root:P(h) Updating Regret? False +DEBUG:root:Betting Round terminal +DEBUG:root:Community Cards [, , , , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): default info set, please ensure you load it correctly +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:Got EV for fold: -200 +DEBUG:root:Added to Node EV for ACTION: fold INFOSET: {"cards_cluster":25,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call","raise"]}]} + STRATEGY: 0.3333333333333333: -66.66666666666666 +DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: call +DEBUG:pluribus.games.short_deck.state:calling +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 0 +DEBUG:root:P(h) Updating Regret? True +DEBUG:root:Betting Round river +DEBUG:root:Community Cards [, , , , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): {"cards_cluster":10,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call","raise","skip","call"]}]} +DEBUG:root:Betting Action Correct?: [, , ] +DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x10d7aeb70>, {}) +DEBUG:root:Current regret: defaultdict(... at 0x10d7aeb70>, {}) +DEBUG:root:Calculated Strategy for {"cards_cluster":10,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call","raise","skip","call"]}]}: defaultdict(... at 0x10d7aec80>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) +DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: fold +DEBUG:pluribus.poker.engine:f"Rank #1600 straight +DEBUG:pluribus.poker.engine:Winnings computation complete. Players: +DEBUG:pluribus.poker.engine: +DEBUG:pluribus.poker.engine: +DEBUG:pluribus.poker.engine: +DEBUG:root:CFR +DEBUG:root:######## +DEBUG:root:Iteration: 1 +DEBUG:root:Player Set to Update Regret: 0 +DEBUG:root:P(h): 1 +DEBUG:root:P(h) Updating Regret? False +DEBUG:root:Betting Round terminal +DEBUG:root:Community Cards [, , , , ] +DEBUG:root:Player 0 hole cards: [, ] +DEBUG:root:Player 1 hole cards: [, ] +DEBUG:root:Player 2 hole cards: [, ] +DEBUG:root:I(h): default info set, please ensure you load it correctly +DEBUG:root:Betting Action Correct?: [, , ] From 715c4f88b11ff4da137cde2c86f078ad9b5778d1 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Mon, 25 May 2020 10:29:15 -0400 Subject: [PATCH 30/42] removing sample files --- research/blueprint_algo/after_RT.txt | 307 -------------------------- research/blueprint_algo/before_RT.txt | 307 -------------------------- 2 files changed, 614 deletions(-) delete mode 100644 research/blueprint_algo/after_RT.txt delete mode 100644 research/blueprint_algo/before_RT.txt diff --git a/research/blueprint_algo/after_RT.txt b/research/blueprint_algo/after_RT.txt deleted file mode 100644 index b669b4a6..00000000 --- a/research/blueprint_algo/after_RT.txt +++ /dev/null @@ -1,307 +0,0 @@ -DEBUG:pluribus.poker.engine:Assigned blinds to players [, ] -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 2 -DEBUG:root:P(h) Updating Regret? False -DEBUG:root:Betting Round pre_flop -DEBUG:root:Community Cards [] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): {"cards_cluster":18,"history":[]} -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x1133139d8>, {}) -DEBUG:root:Current regret: defaultdict(... at 0x1133139d8>, {}) -DEBUG:root:Calculated Strategy for {"cards_cluster":18,"history":[]}: defaultdict(... at 0x113316378>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) -DEBUG:root:ACTION SAMPLED: ph 2 ACTION: fold -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 0 -DEBUG:root:P(h) Updating Regret? True -DEBUG:root:Betting Round pre_flop -DEBUG:root:Community Cards [] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): {"cards_cluster":24,"history":[{"pre_flop":["fold"]}]} -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x113316268>, {}) -DEBUG:root:Current regret: defaultdict(... at 0x113316268>, {}) -DEBUG:root:Calculated Strategy for {"cards_cluster":24,"history":[{"pre_flop":["fold"]}]}: defaultdict(... at 0x1133161e0>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) -DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: fold -DEBUG:pluribus.poker.engine:f"Rank #3776 pair -DEBUG:pluribus.poker.engine:Winnings computation complete. Players: -DEBUG:pluribus.poker.engine: -DEBUG:pluribus.poker.engine: -DEBUG:pluribus.poker.engine: -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 2 -DEBUG:root:P(h) Updating Regret? False -DEBUG:root:Betting Round terminal -DEBUG:root:Community Cards [, , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): default info set, please ensure you load it correctly -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:Got EV for fold: -50 -DEBUG:root:Added to Node EV for ACTION: fold INFOSET: {"cards_cluster":24,"history":[{"pre_flop":["fold"]}]} - STRATEGY: 0.3333333333333333: -16.666666666666664 -DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: call -DEBUG:pluribus.games.short_deck.state:calling -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 1 -DEBUG:root:P(h) Updating Regret? False -DEBUG:root:Betting Round pre_flop -DEBUG:root:Community Cards [] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): {"cards_cluster":22,"history":[{"pre_flop":["fold","call"]}]} -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x113316158>, {}) -DEBUG:root:Current regret: defaultdict(... at 0x113316158>, {}) -DEBUG:root:Calculated Strategy for {"cards_cluster":22,"history":[{"pre_flop":["fold","call"]}]}: defaultdict(... at 0x1133160d0>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) -DEBUG:root:ACTION SAMPLED: ph 1 ACTION: call -DEBUG:pluribus.games.short_deck.state:calling -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 0 -DEBUG:root:P(h) Updating Regret? True -DEBUG:root:Betting Round flop -DEBUG:root:Community Cards [, , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): {"cards_cluster":1,"history":[{"pre_flop":["fold","call","call"]}]} -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x113316950>, {}) -DEBUG:root:Current regret: defaultdict(... at 0x113316950>, {}) -DEBUG:root:Calculated Strategy for {"cards_cluster":1,"history":[{"pre_flop":["fold","call","call"]}]}: defaultdict(... at 0x1133169d8>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) -DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: fold -DEBUG:pluribus.poker.engine:f"Rank #3986 pair -DEBUG:pluribus.poker.engine:Winnings computation complete. Players: -DEBUG:pluribus.poker.engine: -DEBUG:pluribus.poker.engine: -DEBUG:pluribus.poker.engine: -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 1 -DEBUG:root:P(h) Updating Regret? False -DEBUG:root:Betting Round terminal -DEBUG:root:Community Cards [, , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): default info set, please ensure you load it correctly -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:Got EV for fold: -100 -DEBUG:root:Added to Node EV for ACTION: fold INFOSET: {"cards_cluster":1,"history":[{"pre_flop":["fold","call","call"]}]} - STRATEGY: 0.3333333333333333: -33.33333333333333 -DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: call -DEBUG:pluribus.games.short_deck.state:calling -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 1 -DEBUG:root:P(h) Updating Regret? False -DEBUG:root:Betting Round flop -DEBUG:root:Community Cards [, , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): {"cards_cluster":22,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call"]}]} -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x113316ae8>, {}) -DEBUG:root:Current regret: defaultdict(... at 0x113316ae8>, {}) -DEBUG:root:Calculated Strategy for {"cards_cluster":22,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call"]}]}: defaultdict(... at 0x113316bf8>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) -DEBUG:root:ACTION SAMPLED: ph 1 ACTION: raise -DEBUG:pluribus.games.short_deck.state:betting 100 n chips -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 0 -DEBUG:root:P(h) Updating Regret? True -DEBUG:root:Betting Round flop -DEBUG:root:Community Cards [, , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): {"cards_cluster":1,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise"]}]} -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x113316d08>, {}) -DEBUG:root:Current regret: defaultdict(... at 0x113316d08>, {}) -DEBUG:root:Calculated Strategy for {"cards_cluster":1,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise"]}]}: defaultdict(... at 0x113316d90>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) -DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: fold -DEBUG:pluribus.poker.engine:f"Rank #2721 two pair -DEBUG:pluribus.poker.engine:Winnings computation complete. Players: -DEBUG:pluribus.poker.engine: -DEBUG:pluribus.poker.engine: -DEBUG:pluribus.poker.engine: -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 1 -DEBUG:root:P(h) Updating Regret? False -DEBUG:root:Betting Round terminal -DEBUG:root:Community Cards [, , , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): default info set, please ensure you load it correctly -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:Got EV for fold: -100 -DEBUG:root:Added to Node EV for ACTION: fold INFOSET: {"cards_cluster":1,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise"]}]} - STRATEGY: 0.3333333333333333: -33.33333333333333 -DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: call -DEBUG:pluribus.games.short_deck.state:calling -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 0 -DEBUG:root:P(h) Updating Regret? True -DEBUG:root:Betting Round turn -DEBUG:root:Community Cards [, , , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): {"cards_cluster":24,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]}]} -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x113316e18>, {}) -DEBUG:root:Current regret: defaultdict(... at 0x113316e18>, {}) -DEBUG:root:Calculated Strategy for {"cards_cluster":24,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]}]}: defaultdict(... at 0x113316f28>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) -DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: fold -DEBUG:pluribus.poker.engine:f"Rank #2721 two pair -DEBUG:pluribus.poker.engine:Winnings computation complete. Players: -DEBUG:pluribus.poker.engine: -DEBUG:pluribus.poker.engine: -DEBUG:pluribus.poker.engine: -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 1 -DEBUG:root:P(h) Updating Regret? False -DEBUG:root:Betting Round terminal -DEBUG:root:Community Cards [, , , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): default info set, please ensure you load it correctly -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:Got EV for fold: -200 -DEBUG:root:Added to Node EV for ACTION: fold INFOSET: {"cards_cluster":24,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]}]} - STRATEGY: 0.3333333333333333: -66.66666666666666 -DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: call -DEBUG:pluribus.games.short_deck.state:calling -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 1 -DEBUG:root:P(h) Updating Regret? False -DEBUG:root:Betting Round turn -DEBUG:root:Community Cards [, , , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): {"cards_cluster":23,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call"]}]} -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x113d01048>, {}) -DEBUG:root:Current regret: defaultdict(... at 0x113d01048>, {}) -DEBUG:root:Calculated Strategy for {"cards_cluster":23,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call"]}]}: defaultdict(... at 0x113d01158>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) -DEBUG:root:ACTION SAMPLED: ph 1 ACTION: raise -DEBUG:pluribus.games.short_deck.state:betting 200 n chips -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 0 -DEBUG:root:P(h) Updating Regret? True -DEBUG:root:Betting Round turn -DEBUG:root:Community Cards [, , , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): {"cards_cluster":24,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call","raise"]}]} -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x113d01268>, {}) -DEBUG:root:Current regret: defaultdict(... at 0x113d01268>, {}) -DEBUG:root:Calculated Strategy for {"cards_cluster":24,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call","raise"]}]}: defaultdict(... at 0x113d012f0>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) -DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: fold -DEBUG:pluribus.poker.engine:f"Rank #205 full house -DEBUG:pluribus.poker.engine:Winnings computation complete. Players: -DEBUG:pluribus.poker.engine: -DEBUG:pluribus.poker.engine: -DEBUG:pluribus.poker.engine: -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 1 -DEBUG:root:P(h) Updating Regret? False -DEBUG:root:Betting Round terminal -DEBUG:root:Community Cards [, , , , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): default info set, please ensure you load it correctly -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:Got EV for fold: -200 -DEBUG:root:Added to Node EV for ACTION: fold INFOSET: {"cards_cluster":24,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call","raise"]}]} - STRATEGY: 0.3333333333333333: -66.66666666666666 -DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: call -DEBUG:pluribus.games.short_deck.state:calling -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 0 -DEBUG:root:P(h) Updating Regret? True -DEBUG:root:Betting Round river -DEBUG:root:Community Cards [, , , , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): {"cards_cluster":10,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call","raise","skip","call"]}]} -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x113d01378>, {}) -DEBUG:root:Current regret: defaultdict(... at 0x113d01378>, {}) -DEBUG:root:Calculated Strategy for {"cards_cluster":10,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call","raise","skip","call"]}]}: defaultdict(... at 0x113d01488>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) -DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: fold -DEBUG:pluribus.poker.engine:f"Rank #1600 straight -DEBUG:pluribus.poker.engine:Winnings computation complete. Players: -DEBUG:pluribus.poker.engine: -DEBUG:pluribus.poker.engine: -DEBUG:pluribus.poker.engine: -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 1 -DEBUG:root:P(h) Updating Regret? False -DEBUG:root:Betting Round terminal -DEBUG:root:Community Cards [, , , , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): default info set, please ensure you load it correctly -DEBUG:root:Betting Action Correct?: [, , ] diff --git a/research/blueprint_algo/before_RT.txt b/research/blueprint_algo/before_RT.txt deleted file mode 100644 index 0ec3e222..00000000 --- a/research/blueprint_algo/before_RT.txt +++ /dev/null @@ -1,307 +0,0 @@ -DEBUG:pluribus.poker.engine:Assigned blinds to players [, ] -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 2 -DEBUG:root:P(h) Updating Regret? False -DEBUG:root:Betting Round pre_flop -DEBUG:root:Community Cards [] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): {"cards_cluster":18,"history":[]} -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x10d79ed90>, {}) -DEBUG:root:Current regret: defaultdict(... at 0x10d79ed90>, {}) -DEBUG:root:Calculated Strategy for {"cards_cluster":18,"history":[]}: defaultdict(... at 0x10d79ed08>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) -DEBUG:root:ACTION SAMPLED: ph 2 ACTION: fold -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 0 -DEBUG:root:P(h) Updating Regret? True -DEBUG:root:Betting Round pre_flop -DEBUG:root:Community Cards [] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): {"cards_cluster":24,"history":[{"pre_flop":["fold"]}]} -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x10d79ebf8>, {}) -DEBUG:root:Current regret: defaultdict(... at 0x10d79ebf8>, {}) -DEBUG:root:Calculated Strategy for {"cards_cluster":24,"history":[{"pre_flop":["fold"]}]}: defaultdict(... at 0x10d79eb70>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) -DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: fold -DEBUG:pluribus.poker.engine:f"Rank #3776 pair -DEBUG:pluribus.poker.engine:Winnings computation complete. Players: -DEBUG:pluribus.poker.engine: -DEBUG:pluribus.poker.engine: -DEBUG:pluribus.poker.engine: -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 2 -DEBUG:root:P(h) Updating Regret? False -DEBUG:root:Betting Round terminal -DEBUG:root:Community Cards [, , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): default info set, please ensure you load it correctly -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:Got EV for fold: -50 -DEBUG:root:Added to Node EV for ACTION: fold INFOSET: {"cards_cluster":24,"history":[{"pre_flop":["fold"]}]} - STRATEGY: 0.3333333333333333: -16.666666666666664 -DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: call -DEBUG:pluribus.games.short_deck.state:calling -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 1 -DEBUG:root:P(h) Updating Regret? False -DEBUG:root:Betting Round pre_flop -DEBUG:root:Community Cards [] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): {"cards_cluster":22,"history":[{"pre_flop":["fold","call"]}]} -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x10d79eae8>, {}) -DEBUG:root:Current regret: defaultdict(... at 0x10d79eae8>, {}) -DEBUG:root:Calculated Strategy for {"cards_cluster":22,"history":[{"pre_flop":["fold","call"]}]}: defaultdict(... at 0x10d79e9d8>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) -DEBUG:root:ACTION SAMPLED: ph 1 ACTION: call -DEBUG:pluribus.games.short_deck.state:calling -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 0 -DEBUG:root:P(h) Updating Regret? True -DEBUG:root:Betting Round flop -DEBUG:root:Community Cards [, , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): {"cards_cluster":14,"history":[{"pre_flop":["fold","call","call"]}]} -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x10d79e950>, {}) -DEBUG:root:Current regret: defaultdict(... at 0x10d79e950>, {}) -DEBUG:root:Calculated Strategy for {"cards_cluster":14,"history":[{"pre_flop":["fold","call","call"]}]}: defaultdict(... at 0x10d7ae268>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) -DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: fold -DEBUG:pluribus.poker.engine:f"Rank #3986 pair -DEBUG:pluribus.poker.engine:Winnings computation complete. Players: -DEBUG:pluribus.poker.engine: -DEBUG:pluribus.poker.engine: -DEBUG:pluribus.poker.engine: -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 1 -DEBUG:root:P(h) Updating Regret? False -DEBUG:root:Betting Round terminal -DEBUG:root:Community Cards [, , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): default info set, please ensure you load it correctly -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:Got EV for fold: -100 -DEBUG:root:Added to Node EV for ACTION: fold INFOSET: {"cards_cluster":14,"history":[{"pre_flop":["fold","call","call"]}]} - STRATEGY: 0.3333333333333333: -33.33333333333333 -DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: call -DEBUG:pluribus.games.short_deck.state:calling -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 1 -DEBUG:root:P(h) Updating Regret? False -DEBUG:root:Betting Round flop -DEBUG:root:Community Cards [, , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): {"cards_cluster":22,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call"]}]} -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x10d7ae378>, {}) -DEBUG:root:Current regret: defaultdict(... at 0x10d7ae378>, {}) -DEBUG:root:Calculated Strategy for {"cards_cluster":22,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call"]}]}: defaultdict(... at 0x10d7ae488>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) -DEBUG:root:ACTION SAMPLED: ph 1 ACTION: raise -DEBUG:pluribus.games.short_deck.state:betting 100 n chips -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 0 -DEBUG:root:P(h) Updating Regret? True -DEBUG:root:Betting Round flop -DEBUG:root:Community Cards [, , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): {"cards_cluster":14,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise"]}]} -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x10d7ae598>, {}) -DEBUG:root:Current regret: defaultdict(... at 0x10d7ae598>, {}) -DEBUG:root:Calculated Strategy for {"cards_cluster":14,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise"]}]}: defaultdict(... at 0x10d7ae620>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) -DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: fold -DEBUG:pluribus.poker.engine:f"Rank #2721 two pair -DEBUG:pluribus.poker.engine:Winnings computation complete. Players: -DEBUG:pluribus.poker.engine: -DEBUG:pluribus.poker.engine: -DEBUG:pluribus.poker.engine: -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 1 -DEBUG:root:P(h) Updating Regret? False -DEBUG:root:Betting Round terminal -DEBUG:root:Community Cards [, , , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): default info set, please ensure you load it correctly -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:Got EV for fold: -100 -DEBUG:root:Added to Node EV for ACTION: fold INFOSET: {"cards_cluster":14,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise"]}]} - STRATEGY: 0.3333333333333333: -33.33333333333333 -DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: call -DEBUG:pluribus.games.short_deck.state:calling -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 0 -DEBUG:root:P(h) Updating Regret? True -DEBUG:root:Betting Round turn -DEBUG:root:Community Cards [, , , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): {"cards_cluster":25,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]}]} -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x10d7ae6a8>, {}) -DEBUG:root:Current regret: defaultdict(... at 0x10d7ae6a8>, {}) -DEBUG:root:Calculated Strategy for {"cards_cluster":25,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]}]}: defaultdict(... at 0x10d7ae7b8>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) -DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: fold -DEBUG:pluribus.poker.engine:f"Rank #2721 two pair -DEBUG:pluribus.poker.engine:Winnings computation complete. Players: -DEBUG:pluribus.poker.engine: -DEBUG:pluribus.poker.engine: -DEBUG:pluribus.poker.engine: -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 1 -DEBUG:root:P(h) Updating Regret? False -DEBUG:root:Betting Round terminal -DEBUG:root:Community Cards [, , , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): default info set, please ensure you load it correctly -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:Got EV for fold: -200 -DEBUG:root:Added to Node EV for ACTION: fold INFOSET: {"cards_cluster":25,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]}]} - STRATEGY: 0.3333333333333333: -66.66666666666666 -DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: call -DEBUG:pluribus.games.short_deck.state:calling -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 1 -DEBUG:root:P(h) Updating Regret? False -DEBUG:root:Betting Round turn -DEBUG:root:Community Cards [, , , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): {"cards_cluster":13,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call"]}]} -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x10d7ae840>, {}) -DEBUG:root:Current regret: defaultdict(... at 0x10d7ae840>, {}) -DEBUG:root:Calculated Strategy for {"cards_cluster":13,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call"]}]}: defaultdict(... at 0x10d7ae950>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) -DEBUG:root:ACTION SAMPLED: ph 1 ACTION: raise -DEBUG:pluribus.games.short_deck.state:betting 200 n chips -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 0 -DEBUG:root:P(h) Updating Regret? True -DEBUG:root:Betting Round turn -DEBUG:root:Community Cards [, , , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): {"cards_cluster":25,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call","raise"]}]} -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x10d7aea60>, {}) -DEBUG:root:Current regret: defaultdict(... at 0x10d7aea60>, {}) -DEBUG:root:Calculated Strategy for {"cards_cluster":25,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call","raise"]}]}: defaultdict(... at 0x10d7aeae8>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) -DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: fold -DEBUG:pluribus.poker.engine:f"Rank #205 full house -DEBUG:pluribus.poker.engine:Winnings computation complete. Players: -DEBUG:pluribus.poker.engine: -DEBUG:pluribus.poker.engine: -DEBUG:pluribus.poker.engine: -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 1 -DEBUG:root:P(h) Updating Regret? False -DEBUG:root:Betting Round terminal -DEBUG:root:Community Cards [, , , , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): default info set, please ensure you load it correctly -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:Got EV for fold: -200 -DEBUG:root:Added to Node EV for ACTION: fold INFOSET: {"cards_cluster":25,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call","raise"]}]} - STRATEGY: 0.3333333333333333: -66.66666666666666 -DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: call -DEBUG:pluribus.games.short_deck.state:calling -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 0 -DEBUG:root:P(h) Updating Regret? True -DEBUG:root:Betting Round river -DEBUG:root:Community Cards [, , , , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): {"cards_cluster":10,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call","raise","skip","call"]}]} -DEBUG:root:Betting Action Correct?: [, , ] -DEBUG:root:About to Calculate Strategy, Regret: defaultdict(... at 0x10d7aeb70>, {}) -DEBUG:root:Current regret: defaultdict(... at 0x10d7aeb70>, {}) -DEBUG:root:Calculated Strategy for {"cards_cluster":10,"history":[{"pre_flop":["fold","call","call"]},{"flop":["call","raise","skip","call"]},{"turn":["call","raise","skip","call"]}]}: defaultdict(... at 0x10d7aec80>, {'fold': 0.3333333333333333, 'call': 0.3333333333333333, 'raise': 0.3333333333333333}) -DEBUG:root:ACTION TRAVERSED FOR REGRET: ph 0 ACTION: fold -DEBUG:pluribus.poker.engine:f"Rank #1600 straight -DEBUG:pluribus.poker.engine:Winnings computation complete. Players: -DEBUG:pluribus.poker.engine: -DEBUG:pluribus.poker.engine: -DEBUG:pluribus.poker.engine: -DEBUG:root:CFR -DEBUG:root:######## -DEBUG:root:Iteration: 1 -DEBUG:root:Player Set to Update Regret: 0 -DEBUG:root:P(h): 1 -DEBUG:root:P(h) Updating Regret? False -DEBUG:root:Betting Round terminal -DEBUG:root:Community Cards [, , , , ] -DEBUG:root:Player 0 hole cards: [, ] -DEBUG:root:Player 1 hole cards: [, ] -DEBUG:root:Player 2 hole cards: [, ] -DEBUG:root:I(h): default info set, please ensure you load it correctly -DEBUG:root:Betting Action Correct?: [, , ] From 90ae9c79186625b69205d41dd977a1d4da02957e Mon Sep 17 00:00:00 2001 From: big-c-note Date: Mon, 25 May 2020 23:32:10 -0400 Subject: [PATCH 31/42] testing many different RTS configs --- research/test_methodology/RT.py | 21 ++++++++++------- research/test_methodology/RT_cfr.py | 23 ++++++++++++++---- research/test_methodology/bot_test.py | 34 +++++++++++++-------------- 3 files changed, 48 insertions(+), 30 deletions(-) diff --git a/research/test_methodology/RT.py b/research/test_methodology/RT.py index 20d7388b..781dd14b 100644 --- a/research/test_methodology/RT.py +++ b/research/test_methodology/RT.py @@ -7,19 +7,22 @@ if __name__ == "__main__": # We can set public cards or not - public_cards = [Card("king", "spades"), Card("queen", "clubs"), - Card("jack", "hearts")] + public_cards = [Card("ace", "diamonds"), Card("king", "clubs"), + Card("jack", "spades"), Card("10", "hearts"), + Card("10", "spades")] # public_cards: List[Card] = [] # Action sequence must be in old form (one list, includes skips) - action_sequence = ["raise", "call", "call", "call"] + action_sequence = ["raise", "raise", "raise", "call", "call", + "raise", "raise", "raise", "call", "call", + "raise", "raise", "raise", "call", "call", "call"] agent_output, offline_strategy = rts( - 'test_strategy/unnormalized_output/offline_strategy_100.gz', - 'test_strategy/strategy_100.gz', public_cards, action_sequence, - 140, 6, 6, 3, 2, 6, 10 + 'test_strategy2/unnormalized_output/offline_strategy_1500.gz', + 'test_strategy2/strategy_1500.gz', public_cards, action_sequence, + 1400, 1, 1, 3, 1, 1, 20 ) - save_path = "test_strategy/unnormalized_output/" + save_path = "test_strategy2/unnormalized_output/" last_regret = {info_set: dict(strategy) for info_set, strategy in agent_output.regret.items()} - joblib.dump(offline_strategy, save_path + 'rts_output.gz', compress="gzip") - joblib.dump(last_regret, save_path + 'last_regret.gz', compress="gzip") + joblib.dump(offline_strategy, save_path + 'rts_output3.gz', compress="gzip") + joblib.dump(last_regret, save_path + 'last_regret3.gz', compress="gzip") import ipdb; ipdb.set_trace() diff --git a/research/test_methodology/RT_cfr.py b/research/test_methodology/RT_cfr.py index 3a43bd2a..4cf69d48 100644 --- a/research/test_methodology/RT_cfr.py +++ b/research/test_methodology/RT_cfr.py @@ -1,12 +1,14 @@ from __future__ import annotations -import logging import collections from typing import Dict import joblib +from pathlib import Path from tqdm import trange import numpy as np +import datetime +import yaml from pluribus import utils from pluribus.games.short_deck.state import ShortDeckPokerState, new_game @@ -51,6 +53,16 @@ def calculate_strategy( return sigma +def _create_dir(folder_id: str) -> Path: + """Create and get a unique dir path to save to using a timestamp.""" + time = str(datetime.datetime.now()) + for char in ":- .": + time = time.replace(char, "_") + path: Path = Path(f"./{folder_id}_results_{time}") + path.mkdir(parents=True, exist_ok=True) + return path + + def cfr(agent: Agent, state: ShortDeckPokerState, i: int, t: int) -> float: """ CFR algo with the a temporary regret object for better strategy averaging @@ -90,6 +102,7 @@ def cfr(agent: Agent, state: ShortDeckPokerState, i: int, t: int) -> float: a = np.random.choice( list(sigma[Iph].keys()), 1, p=list(sigma[Iph].values()), )[0] + except KeyError: p = 1 / len(state.legal_actions) probabilities = np.full(len(state.legal_actions), p) @@ -114,8 +127,12 @@ def rts( dump_int: int, ): """RTS.""" + config: Dict[str, int] = {**locals()} + save_path: Path = _create_dir('RTS') + with open(save_path / "config.yaml", "w") as steam: + yaml.dump(config, steam) # TODO: fix the seed - utils.random.seed(36) + # utils.random.seed(36) agent = Agent(regret_path=regret_path) # Load unnormalized strategy to build off offline_strategy = joblib.load(offline_strategy_path) @@ -129,7 +146,6 @@ def rts( # We don't need the offline strategy for search.. # del offline_strategy for t in trange(1, n_iterations + 1, desc="train iter"): - print(t) for i in range(n_players): # fixed position i # Deal hole cards based on bayesian updating of hole card probs state: ShortDeckPokerState = current_game_state.deal_bayes() @@ -162,5 +178,4 @@ def rts( ipdb.set_trace() agent.reset_new_regret() - return agent, offline_strategy diff --git a/research/test_methodology/bot_test.py b/research/test_methodology/bot_test.py index 376f7a9a..fe5ae245 100644 --- a/research/test_methodology/bot_test.py +++ b/research/test_methodology/bot_test.py @@ -19,9 +19,6 @@ def _calculate_strategy( strategy: DefaultDict[str, DefaultDict[str, float]] ) -> str: sigma = collections.defaultdict(lambda: collections.defaultdict(lambda: 1 / 3)) - import ipdb; - ipdb.set_trace() - try: # If strategy is empty, go to other block if sigma[I] == {}: @@ -41,12 +38,12 @@ def _calculate_strategy( return a -def _create_dir() -> Path: +def _create_dir(folder_id: str) -> Path: """Create and get a unique dir path to save to using a timestamp.""" time = str(datetime.datetime.now()) for char in ":- .": time = time.replace(char, "_") - path: Path = Path(f"./results_{time}") + path: Path = Path(f"./{folder_id}_results_{time}") path.mkdir(parents=True, exist_ok=True) return path @@ -62,7 +59,7 @@ def agent_test( n_players: int = 3, ): config: Dict[str, int] = {**locals()} - save_path: Path = _create_dir() + save_path: Path = _create_dir('bt') with open(save_path / "config.yaml", "w") as steam: yaml.dump(config, steam) @@ -85,7 +82,7 @@ def agent_test( # should be used with caution. It will work only if the probability of reach # is identical across strategies. Use the average strategy. - # Load current game state, either strategy should be identical for this + info_set_lut = {} EVs = np.array([]) for _ in trange(1, n_outter_iters): EV = np.array([]) # Expected value for player 0 (hero) @@ -95,7 +92,8 @@ def agent_test( # Deal hole cards based on bayesian updating of hole card probs state: ShortDeckPokerState = current_game_state.deal_bayes() else: - state: ShortDeckPokerState = new_game(n_players) + state: ShortDeckPokerState = new_game(n_players, info_set_lut) + info_set_lut = state.info_set_lut while True: player_not_in_hand = not state.players[p_i].is_active if state.is_terminal or player_not_in_hand: @@ -120,20 +118,22 @@ def agent_test( results_dict = { 'Expected Value': float(EVs.mean()), 'T Statistic': float(t_stat), - 'P Value': float(p_val) + 'P Value': float(p_val), + 'Standard Deviation': float(EVs.std()), + 'N': int(len(EVs)) } with open(save_path / 'results.yaml', "w") as stream: yaml.safe_dump(results_dict, stream=stream, default_flow_style=False) if __name__ == "__main__": - strat_path = "test_strategy/unnormalized_output/" + strat_path = "test_strategy2/unnormalized_output/" agent_test( - hero_strategy_path=strat_path + "rts_output.gz", - opponent_strategy_path=strat_path + "offline_strategy_100.gz", - real_time_est=True, - public_cards=[Card("ace", "spades"), Card("queen", "spades"), - Card("queen", "hearts")], - action_sequence=["raise", "call", "call", "call", "call"], - n_inner_iters=100 + hero_strategy_path=strat_path + "offline_strategy_1500.gz", + opponent_strategy_path=strat_path + "random_strategy.gz", + real_time_est=False, + public_cards=[], + action_sequence=None, + n_inner_iters=25, + n_outter_iters=50, ) From 5cdd94fdeb12e6ddec62e826fd9a9da792722440 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Tue, 26 May 2020 20:41:52 -0400 Subject: [PATCH 32/42] bug fix for dealing bayes hole cards before next community card --- pluribus/games/short_deck/state.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/pluribus/games/short_deck/state.py b/pluribus/games/short_deck/state.py index fc5e0db7..fa37c0f5 100644 --- a/pluribus/games/short_deck/state.py +++ b/pluribus/games/short_deck/state.py @@ -126,6 +126,7 @@ def __init__( if public_cards: assert len(public_cards) in {3, 4, 5} self._public_cards = public_cards + self._final_action = None # only want to do these actions in real game play, as they are slow if self.real_time_test: # must have offline strategy loaded up @@ -231,27 +232,29 @@ def load_game_state(self, offline_strategy: Dict[str, Dict[str, float]], Follow through the action sequence provided to get current node. :param action_sequence: List of actions without 'skip' """ - if not action_sequence: + if 'skip' in set(action_sequence): + action_sequence = [a for a in action_sequence if a != 'skip'] + if len(action_sequence) == 1: # TODO: Not sure if I need to deepcopy lut = self.info_set_lut self.info_set_lut = {} new_state = copy.deepcopy(self) new_state.info_set_lut = self.info_set_lut = lut + new_state._final_action = action_sequence.pop(0) new_state._update_hole_cards_bayes(offline_strategy) return new_state a = action_sequence.pop(0) - if a == "skip": - a = action_sequence.pop(0) new_state = self.apply_action(a) return new_state.load_game_state(offline_strategy, action_sequence) def deal_bayes(self): - # TODO: Not sure if I need to deepcopy + # Deep copy the parts of state that are needed that must be immutable + # from state to state. lut = self.info_set_lut self.info_set_lut = {} new_state = copy.deepcopy(self) new_state.info_set_lut = self.info_set_lut = lut - players = list(range(len(self.players))) + players = list(range(len(new_state.players))) random.shuffle(players) cards_selected = [] # TODO: This would be better by selecting the first player's @@ -272,7 +275,9 @@ def deal_bayes(self): cards_selected += new_state._public_cards for card in cards_selected: new_state._table.dealer.deck.remove(card) - return new_state + final_action = new_state._final_action + newest_state = new_state.apply_action(final_action) + return newest_state @staticmethod def load_pickle_files(pickle_dir: str) -> Dict[str, Dict[Tuple[int, ...], str]]: @@ -386,6 +391,7 @@ def _normalize_bayes(self): def _update_hole_cards_bayes(self, offline_strategy: Dict[str, Dict[str, float]]): """Get probability of reach for each starting hand for each player""" + assert self._history n_players = len(self._table.players) player_indices: List[int] = [p_i for p_i in range(n_players)] for p_i in player_indices: From a097810e060f5a1ec008dae1a686b130543c575e Mon Sep 17 00:00:00 2001 From: big-c-note Date: Wed, 27 May 2020 21:48:07 -0400 Subject: [PATCH 33/42] fixing random strategy bug, adding entry script for rng game nodes and testing RTS --- research/test_methodology/bot_test.py | 33 +++++++--- research/test_methodology/entry_rts_test.py | 67 +++++++++++++++++++++ 2 files changed, 90 insertions(+), 10 deletions(-) create mode 100644 research/test_methodology/entry_rts_test.py diff --git a/research/test_methodology/bot_test.py b/research/test_methodology/bot_test.py index fe5ae245..678470e4 100644 --- a/research/test_methodology/bot_test.py +++ b/research/test_methodology/bot_test.py @@ -12,18 +12,19 @@ from pluribus.games.short_deck.state import ShortDeckPokerState, new_game from pluribus.poker.card import Card - def _calculate_strategy( state: ShortDeckPokerState, I: str, - strategy: DefaultDict[str, DefaultDict[str, float]] + strategy: DefaultDict[str, DefaultDict[str, float]], + count=None, + total_count=None ) -> str: sigma = collections.defaultdict(lambda: collections.defaultdict(lambda: 1 / 3)) try: # If strategy is empty, go to other block + sigma[I] = strategy[I].copy() if sigma[I] == {}: raise KeyError - sigma[I] = strategy[I].copy() norm = sum(sigma[I].values()) for a in sigma[I].keys(): sigma[I][a] /= norm @@ -31,11 +32,15 @@ def _calculate_strategy( list(sigma[I].keys()), 1, p=list(sigma[I].values()), )[0] except KeyError: + if count is not None: + count += 1 p = 1 / len(state.legal_actions) probabilities = np.full(len(state.legal_actions), p) a = np.random.choice(state.legal_actions, p=probabilities) sigma[I] = {action: p for action in state.legal_actions} - return a + if total_count is not None: + total_count += 1 + return a, count, total_count def _create_dir(folder_id: str) -> Path: @@ -57,6 +62,8 @@ def agent_test( n_outter_iters: int = 30, n_inner_iters: int = 100, n_players: int = 3, + hero_count=None, + hero_total_count=None, ): config: Dict[str, int] = {**locals()} save_path: Path = _create_dir('bt') @@ -100,13 +107,15 @@ def agent_test( EV = np.append(EV, state.payout[p_i]) break if state.player_i == p_i: - random_action: str = _calculate_strategy( + random_action, hero_count, hero_total_count = _calculate_strategy( state, state.info_set, hero_strategy, + count=hero_count, + total_count=hero_total_count ) else: - random_action: str = _calculate_strategy( + random_action, oc, otc = _calculate_strategy( state, state.info_set, opponent_strategy, @@ -120,7 +129,9 @@ def agent_test( 'T Statistic': float(t_stat), 'P Value': float(p_val), 'Standard Deviation': float(EVs.std()), - 'N': int(len(EVs)) + 'N': int(len(EVs)), + 'Random Moves Hero': hero_count, + 'Total Moves Hero': hero_total_count } with open(save_path / 'results.yaml', "w") as stream: yaml.safe_dump(results_dict, stream=stream, default_flow_style=False) @@ -129,11 +140,13 @@ def agent_test( if __name__ == "__main__": strat_path = "test_strategy2/unnormalized_output/" agent_test( - hero_strategy_path=strat_path + "offline_strategy_1500.gz", - opponent_strategy_path=strat_path + "random_strategy.gz", + hero_strategy_path=strat_path + "random_strategy.gz", + opponent_strategy_path=strat_path + "offline_strategy_1500.gz", real_time_est=False, public_cards=[], action_sequence=None, n_inner_iters=25, - n_outter_iters=50, + n_outter_iters=75, + hero_count=0, + hero_total_count=0 ) diff --git a/research/test_methodology/entry_rts_test.py b/research/test_methodology/entry_rts_test.py new file mode 100644 index 00000000..df29136e --- /dev/null +++ b/research/test_methodology/entry_rts_test.py @@ -0,0 +1,67 @@ +import numpy as np +import json +import joblib + +from RT_cfr import rts +from bot_test import agent_test +from pluribus.poker.deck import Deck + + +check = joblib.load('test_strategy2/unnormalized_output/offline_strategy_1500.gz') +histories = np.random.choice(list(check.keys()), 2) +action_sequences = [] +public_cards_lst = [] +community_card_dict = { + "pre_flop": 0, + "flop": 3, + "turn": 4, + "river": 5, +} +ranks = list(range(10, 14 + 1)) +deck = Deck(include_ranks=ranks) +for history in histories: + history_dict = json.loads(history) + history_lst = history_dict['history'] + action_sequence = [] + betting_rounds = [] + for x in history_lst: + action_sequence += list(x.values())[0] + betting_rounds += list(x.keys()) + action_sequences.append(action_sequence) + final_betting_round = list(betting_rounds)[-1] + n_cards = community_card_dict[final_betting_round] + cards_in_deck = deck._cards_in_deck + public_cards = np.random.choice(cards_in_deck, n_cards) + public_cards_lst.append(list(public_cards)) + +for i in range(0, len(action_sequences)): +# try: + public_cards = public_cards_lst[i].copy() + action_sequence = action_sequences[i].copy() + agent_output, offline_strategy = rts( + 'test_strategy2/unnormalized_output/offline_strategy_1500.gz', + 'test_strategy2/strategy_1500.gz', public_cards, action_sequence, + 1500, 400, 400, 3, 400, 400, 20 + ) + save_path = "test_strategy2/unnormalized_output/" + last_regret = {info_set: dict(strategy) for info_set, strategy in agent_output.regret.items()} + joblib.dump(offline_strategy, save_path + f'rts_output{i+35}.gz', compress="gzip") + joblib.dump(last_regret, save_path + f'last_regret{i+35}.gz', compress="gzip") + + public_cards = public_cards_lst[i].copy() + action_sequence = action_sequences[i].copy() + strat_path = "test_strategy2/unnormalized_output/" + agent_test( + hero_strategy_path=strat_path + f"rts_output{i+35}.gz", + opponent_strategy_path=strat_path + "offline_strategy_1500.gz", + real_time_est=True, + public_cards=public_cards, + action_sequence=action_sequence, + n_inner_iters=25, + n_outter_iters=250, + hero_count=0, + hero_total_count=0, + ) +# except: +# print(f"ERROR on {action_sequences[i]}, {public_cards_lst[i]}") +# pass From e82eb4129e1e51dc9ce0813a837e4a67adfd2b75 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Wed, 27 May 2020 21:55:28 -0400 Subject: [PATCH 34/42] fixing pytest build fail --- research/test_methodology/{entry_rts_test.py => entry_rts_ab.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename research/test_methodology/{entry_rts_test.py => entry_rts_ab.py} (100%) diff --git a/research/test_methodology/entry_rts_test.py b/research/test_methodology/entry_rts_ab.py similarity index 100% rename from research/test_methodology/entry_rts_test.py rename to research/test_methodology/entry_rts_ab.py From 7b4b777b224b6835550e9208c8a7326b459ad42b Mon Sep 17 00:00:00 2001 From: big-c-note Date: Sat, 30 May 2020 20:37:48 -0400 Subject: [PATCH 35/42] updating to decent defaults --- .../blueprint_short_deck_poker.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/research/blueprint_algo/blueprint_short_deck_poker.py b/research/blueprint_algo/blueprint_short_deck_poker.py index b2deeb34..ca3f89bb 100644 --- a/research/blueprint_algo/blueprint_short_deck_poker.py +++ b/research/blueprint_algo/blueprint_short_deck_poker.py @@ -204,7 +204,7 @@ def cfr(agent: Agent, state: ShortDeckPokerState, i: int, t: int) -> float: logging.debug(f"Got EV for {a}: {voa[a]}") vo += sigma[I][a] * voa[a] logging.debug( - f"""Added to Node EV for ACTION: {a} INFOSET: {I} + f"""Added to Node EV for ACTION: {a} INFOSET: {I} STRATEGY: {sigma[I][a]}: {sigma[I][a] * voa[a]}""" ) logging.debug(f"Updated EV at {I}: {vo}") @@ -346,16 +346,16 @@ def _create_dir() -> Path: @click.command() -@click.option("--strategy_interval", default=2, help=".") -@click.option("--n_iterations", default=10, help=".") -@click.option("--lcfr_threshold", default=80, help=".") -@click.option("--discount_interval", default=1000, help=".") -@click.option("--prune_threshold", default=4000, help=".") +@click.option("--strategy_interval", default=400, help=".") +@click.option("--n_iterations", default=5500, help=".") +@click.option("--lcfr_threshold", default=400, help=".") +@click.option("--discount_interval", default=400, help=".") +@click.option("--prune_threshold", default=400, help=".") @click.option("--c", default=-20000, help=".") @click.option("--n_players", default=3, help=".") -@click.option("--print_iteration", default=10, help=".") -@click.option("--dump_iteration", default=10, help=".") -@click.option("--update_threshold", default=0, help=".") +@click.option("--print_iteration", default=100, help=".") +@click.option("--dump_iteration", default=20, help=".") +@click.option("--update_threshold", default=400, help=".") def train( strategy_interval: int, n_iterations: int, From 4d70d20d6563b1ba4976ca4af5cc3ba9171e8ca9 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Sun, 31 May 2020 22:39:30 -0400 Subject: [PATCH 36/42] regression-style test for loading game history --- .../random_action_sequences.pkl | Bin 0 -> 51869 bytes test/functional/test_short_deck.py | 32 +++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 research/size_of_problem/random_action_sequences.pkl diff --git a/research/size_of_problem/random_action_sequences.pkl b/research/size_of_problem/random_action_sequences.pkl new file mode 100644 index 0000000000000000000000000000000000000000..41ab5a88608847ed7514bac1817cd8f67c41683a GIT binary patch literal 51869 zcma)_1+-;Hv4$st1-B5~Ay{yCmq37ELBbFjNFa766Ck*S;O_43?(XjH?(X{P@2~%^ z>V3}5jB%hA8_n$8(a3A zHt(`+$JPUO?AX5B#!l1vp@$!J^Nj&m*X~e-FM?e`<)_yTeojJ=*T0E-m(3dgLZ73_@Nso z*>~fl2Q~I?M;zRPoNVtc$8GF(?VS$XIQj7#r`U7voo@28kT_+B9lYb%qr&%8`|TWZ zH@(8fsrTNp_fE%eoMzvR)9$(VF8l2itM}PB-9G#5bEE&pf3>f@ufAjB^annC|BW;3 zx&Ou)_uOySz#MeIVTWz(e#q|oohT;T4>C)XF(u0ErJf5?Dl*D=8ttJ^IGkI-mYS<` zB;2i-1@%00O+czxuE-6)^R8ATN`_M1DDnmE%H}#s-4Ri=J?*(l0PgV2XuOCQ!q1T?pYJNw`x%4_uP2xyW%_rK zb&>L92%>K_`=VA|9-M>CVTxOjFQz-dmCy#zd`f?>)Q7uXH{CVznOt1@5LU^0PhvMb zF0sx7qQlS~;Fnx|tpHPr#w|oBUkYWmiQq9H$AUylFD;U{O)|~EY=d0YsmmnIV3YM6 z6<9hbmvuZ7l-uJZlZNWluglR^Lo-s2aC93kf5OqS7DhO$rYjIo3yw8=u(L;Nz!f={ zcB-8KiT+$EwJ@NpS=;LBmI7S4lAIIz&~DKLz6t?S-9VsrVkU=HRS>UAf2{8i1a#?o zYDZ32BUWzNNtYU$U|07+vAwEagIW6;Cb6k+k8&)Zn1#nRmql|=OvxG)=vowt4iE$E z!jtYu|JwBgDh1$j#dAH?o)8L=>tuuJnPGrv3}aCg^}5tmfjGL0^NgxRyq?d-fF7fD zv5fZ(g}EVx*D4qjim`wMfD9$o%7KA(bep%J9B-It2R?LHVfSk1DPqsj7b(OhRL_82; zn>KodLX-#f9ORz0CU)KQM5KW`;b6xiPC}~y`azC3+uSh+f?}dq^(34~ceV@L2?E9s z>0OrDtw|L7c$MFkLsGeFckBM|-=Q`0(ppwwx0U<^i``S&RGT7=!%J-YW=%NOh>wJF)z&6tv z3ehiQ%tHGCK8qQ|Oy5o~@BpPx1Ef!pH16RE1x$MD zYRx|wk1!N1faE26QoToWVbT#kmL3xZQ zwg=379*g4F`muxp0M%nok}!`mEGuASu&9Jd5xJ<6$J3!zh$wSF2v2aQQW02{s4M*P z$m@x#R5pIdt^xCNSfChAi; zS^#dbT`i?uAMDe77m`vd_1Qe#HCV&imFBfjk0{SDB_T=Hn*Y3SuV*fAv}h(|IMd|y ztSnoqY(X(KTdebcb|n}nqG05I&bs{8=BaPUxVov&Jz?q`(ZN*5^O9&?KUbY|MbzhS z-nHx*=N^(lc!9lY71U!-J+neJr_F-=!X#%5(>hG83FwQK!BP=*Xn64x_+qOsp?E=S z&2BgCC3av#?e)l!P;aw{^wK2Ez}e4xs^Vpel-osbtsDDsv_XCOlA1D-^`Eq#AvP^G^Xdg@Y+P&AY45`(4OgMSGufnZms^(ng_dB^2fYzB-_&8oGK0H2P zkL9_VkY(c>>p}bAl4dtQ?Z~D0kh_)JL#_3T(^ksUVH{n@6A& z>QknW9at6VK?C|}m)e(_vc}fdlNrZE*`G1Zg=ZiWp;3p9>a#VJHAPD=dh|JIr)`C> zikC5Z@-BSdL?~4`D=NYf>I+b+E^)MC=Xp%XFS-Gl9vD*GtDYcyzC`g>V#6~ib5*YC z%dQD<*0HWeWC{3{O0<&zd_3}wgj?tc`_-i8c8zDU315S-Y=tAXI}o~G&l}(^o=KFl zRCSBLAyr!$$lU9|1^%YAm*7mVk#yr3;BVC+fQVp%g&Hjg-=;tnB4AQ9(p(X~Q_nbI zkd#=~-_71)(l&-Bg;~ENg7&@I3%1umwUcFjf7uhVrf-H!G4)^7{J@Ts2mq@6>(eMv ze@KnlCy`Ax$#Xrqu|M*u0I^~v^xA@=Uj8_HY0#2H2JfcVPh4HbAf)DoTTC_m)V*Z0 zg6eqCgYvWVa09G7V9m`J{^wZ_vG}&Wo=X9r)-UYk%y#tC)V#u~>X)v{e*7cn8IA(} zs_JXoKFZEKo&o&zY7K@v3Qw(aoYMbI(xj(9bd|RV1oXFtR@h8}46OwW+V5O}m7(X% z61i^b?{Rbs36wtAFJ_EaM1%gvIkF{Yf~%C@pG=q`o@D_vo{)bw zX|BU=)TtRH9r9mjh_|Hgk7@Q|-PiQ5CgMf*97AT6U!lM8n~)V$CsuLbC*<#0OWlNo z&wxi{8t@N1(!6?PI15sl+Wk+eajE3?8`MMs^skJR0f45W=2Y-+t_)#N24*rZgihf< ztM<*bokev(J!FFP-({R}u6oh7?Z16X{7*+G#Yp7Uyrly+@5aQlfyWtC+#PbWI@N;Q=; zJJADm{3L^A)z+PIjH>peC4F|pnX6BExwoHeV%7TGGrZa?Tz2dDdLq?L<91csGhBmn z^1*TEsKeP@4X1XBk|^^y&C(05>OW=US(zNLgz-9xn&~6nsaE`!SkF_%9aMFvF4fIQ z&#FJUxNoN^&Z#X{&gQ(C`m}4gM%x*;?49r7=>|LZG)iBMxZkI5Sa#5zQORq&(UCKB z25Iz7zlgo@=l=bIpcY@Xs>b5{Fauf3M-sD*AI4^57}Qiw{TRteYDCO&0zV zo8XtJe8!h-_^RcO_(`48zSO|7=OogWUe6xLOOF((>;yc8okvb|na)MeD67}5ch~T{ z>_}!7zBw~b9NpE+tt7OAQhAIDSr_?3;}ia1hJ9g|b*>WMO5c?~Z;7^R{saN?^Bws&yIncbbKDsn1UZ5nuh7SG$clR8MV>%-}4ge$z!Q$;Kr; zol2~mjcSaV9p;2+&=aw5K9Xjh^a0m)0=sX?CspUOgZ=F>pXe>RQ!mn{9lNT>3TU)Rp^o3a)KfFiuUyeP)v^WW4xLkV-9Mhm^Lw)Y zR7u@ms?18%cE{mpCjD>+c*69%zjX~qHPnQ3`A!3=y{btz*7U889ndSu)vAaBJbpzF z?CREKj~t2iu2$N84X1;;6RB|_!@+%rwM>#-Va*($!nP@kcKzXWnKza2zjJr7s&~m+ zy{U$MPV6qjX=bn1Z5C&7$W$5W?B}dv@Ta`Hoo26dsN}S)qGJHV!*R8BY(X zB!8EV>Mm9j+KP--#m0&*s>zz?T5t6Pcl)Tr+L^>Fa+J@}BMo}qCp8u2*bX~}1HRTL z9jxWA^<%tYi-UJy(1xhfc`f-mx33 zV<#5fK-XhurozS!-n~j%wB@#RWJP-1xqCO3k)UUe_u`Dv3$Oc3tkfjoJcsYozVF1e zlk1}sn#E4|+^?gJlX4yBT&r|aH}U?B=PtkiLAKgS>!IwvJfJ9~%dF^GYLY<7AK3YJ z1(RM?lX^Li2Ti`qoc@K1_3(Ib!{Shtmn?Fsw}A@YLyCua>+;pQgi~jaE8Mw;I$NrV z?an^5D6pP+b5wUy%^Jr~8p2W|$n<(x$1M>#PoK6H6+V1uqgsB~TNQx|)FUQkt27OD zd67t+d*skf)aq?^1=J&?xR09LyjOL!R+0wi(Zw%)`YmX8!F)M9rl6fc`^nTu{V(pp zdu-zw1x5&?tTmK^^SI72s$vE%yjX9Z!Fqg8IH#&y>cP3M2k!}kXT|holyf1}jVDf= zc|TKizra7K+uGZ$8J zcAE4wZq+umEgRfa9saLG4drFH!)v0^4WtYY#6u+{i5NMsN~r>eKcyf zKDaL~PFcONvr4Z#STEVsads2KH{zwk34bG(9<0*~ZpO>H8~KE<`8>92n@7f%cR%kh zeSB8v>5N}deA0^(1%*eeDe$i>71?E(t@SzjE&W$@eK2O!dMf(WYxB=lm@A$fUNaoL zYGpeLIm%%6OX-uXlOI#^SX}DGt7R0%XwD|*6W8$c8L_$PL{eZr#BQ%x-I!m ztlvAa-q={R&-kF=$|?)DH!a+%j}++gn+M8zgP= z!uzI-PBmk5mE_{Re|QR;E*|TdJIw6)fx!iBw~9{&iPg^!HXi9+Vc+duI<>wZ_zz9B zIb&tyn7i@e?vU#w)OS5$rTIuvuXx5rt5+W#s;d>IzIdd^O&%ZX9<`T@E1If>|9Hvd zu39snssD+B*D9)fyMpZJC#O0xEy}_(ie5#QPmSy-TRoqQx_*0OK0P?z6PI^!;9^C8 zs`oPmZ${hem9VCyO7z)|CJ$8#kql=SxnjTPN|Tu-~c< z=PwNAs>iIrtf>X>i&Nt2F;`2CoY4d) zXKm}}lYV_`uqvMSE|ugN%y9X3Nre{4QkS{ZsN*{eMn&RjA9hq*`hB;vwdYSJZ62{x z8Q<#_We)2PRAc0u`2FGGY!%}CUmK_4MD=Ob&ju@1lMlmP_Pzi4 z;Ar2N%BpK^9Ql4RRW*_~1>;$DAAY&k$4of-)~B6jxcsVdlx5a5+N*l1p7;do*Hbq3 z^4xJ?&K@es|IJA1bQPUSb_Er)@c!-a9;=ZJPkeGFN9Fz9@H4{o+JSXpf8Vg)cRus# z-0!tme;BNFNLSRJf&ODhlbT(UkX>WcrTwR#zT)gc&YX3{VEuWpP@XZ&^qs#0e<=(& z>m0#z5B^#h?hFZ1KRanebB%<5Yiy9oV&3j_J5_by?`wUM({A%X|D)?xW%>#u#|;ts z{~Y@4IWx~ZiGK~8nEs@mM(LrE{*eNa+x{ube=q@=48VK)#@RZ6qenH)$rSG!So3@S#%;$C-14y%G|SR$r+qe4CP#x zs!yS04D>07gL7)%DZJ)-PgT$rm1B*pz)76C=kaQ48>?qO3H2;eou>QPKQ)t+-HpnE zbK1cXhK~7J`zJ5qPuFvSwiBv}cj}TcxThaAB@g7PqgHg5XV@fPD&!pqPB6~c8BUm~ zw31F9>8W+<+kMlF*0G8hN_eJ`(25}HOy@+8;u)Mhx{Hwc0b%`t?_94prO-I@325L} z6T_y)vy>X`$$sksiNfKmC*a^Tdin&^jkB$Atej~&C(vhaXnNqxI*U3M+9`cX=P10~ zH_mK!P*b*ZmTc=VBn0pMoJrwaLxDs>fm9ESjg)~scgbiqCo(Jf%{y4<=`1sh$T==# z)m=L8@L6Vp3saqI%wU{vRFPWFP94;uD$ZZ9nI3mE!SYSz7Z{10e#s;$Z}hN3=ZtkO+^Uzx;$37-ee@4EefO>h z`l6$z&3u}!Z(A@fHtAd09d;t8*{kc5+q+KUCwY|q;!_^**&XBOb$9s^gKM0ge7vGL zH#0`zcgbE&-b^a;{Vd*rzEnq7UGU)Sk~5I1?9x4*xwQtbr5n3`i^RXonxsNya%Ngl zKiP@3;9hp&tz8h+*&No4kgEJIw^qaMeu^dZ|2EHUfQF*V_6pTp)5NLU9f8V<+k6Q6F8D@x7e31u}D;9PrfY8Did?LEA76R*>GPU5GTb?$uEEmfs<9l;{m^$OY* zWffYXW9rxS3wG8+QTh2}S4{D4(0NXsJ~^^N)_%jrLP_5~IlAO^>EEa$-FpzyuBvDV zzjw(~F>{*aqC~%LT)62#8fqt}_T+VwQH48WM>uvtOEaY_Z zlSj+dzIDHDHl@0vSO2l`eeRpPS?IGBkN_1t<&Xb)@53jC{bo1V;=S({aT^Pvy=|HbM5gEg6A zRyV_Mh`Mmw;#K`pBX4QzVQ_9Yyo54~G@d0Nj8t@wZhwM)m5rYIrM${|hwhuM>7IQB zyMMuw#oBO~X{-_N2k+q>dqP)WUR43FOj`@unnJdE+ZlN^o`Jv9CisGCiR{+agLgo$ z$2jlPAJ;@K`5f5siGxX>ST$EO@9OD~>N{u?oh3ug=ma}>2ba9F_Jjrg7FWyUyjQJGfE!PUkyR`y;RI9b0vz&75y@xh+Q*b*Hc@&xsu#I|idF zCyLa^j3myYXRE0_jaP+tFW5}ld*0G(RrTGYAyIP5oXvT;cd;A1dlsJX_0RlVSGG0_ z=CM6n*1FCYycLptuTdGj#dmgOrfOB;y%!8LXMMD~OYdga4~6@b#F@IHl|r{bow@Ho zTeT~-9*O0u>ONzd!X z(L3^>HOg}$5{eV+(ZdJ#T2_;@LLz%ol@)d74=Jf?ZjYT3owgdR;|hzu+YKkPV=vGT z?HOwdSo)?r$B#Oes9KrRDEY$%Bi%eQ&e?c_A3iZEHa)fK56K?USYFHA!sTk7$Rk(q zp!BW3WchdfQ6ragq^?(Z!d3N;E*$R}yIQ^V#C%L=3SBdKY&Ff>-3-oSdrcLyIa@pQ z?s0=>rk=^Pm%Hj+*pFXHDke%sx~EcdKcCQf#@Z34Jw5ZBJMzTh$!Yo9c%1Q5Kc6%) zjQZa^sq@KAA=56oB;-4*d`j_!JO0(_J<_}P)Pk}C6}b*~P;+jv)KuBi3YWUzc$|r; zP&+0kiPW`I) z*`2c)!wJcxN}kixEbDMjamQI0`t;mVLq*i_R9QGZuhc`2GJWP{6#0Vj{K*CFL{_g% z_JYE(OCqgN&qRHDVP`GXbIu4wt3+3V`=U{odsbO6#=Q>wi-#9`pgjgDhgMVIU$RmW zrvU~}ecMCzymWYECLEFrhu@5s6_2V*HtNmw?DoqWPjh2<*137=c|%6NS4{b|>U1zY zLroREvO7^vx>(%|l*D*d@j#W@6KQVtL_HMu)t#Ge$?`~S<6krI%UaUI85%!4URyXq z@iE>mxtgK&}DC8cANrn~`y?$U*F-NrnmGy?sqN~nix8+MKVjddmscQa!}xwyMi-EtNFqsDXZvh_xKc>un?1 z+&S|qhiW;|x3A48v_Fe3#E$oT7vC{?+X;ToNJ3}c*(s~?I8Q(o+gl&E@;8qk4xU;NS zdGnh4^N}fCR+9;})=s>o<|IE_SW8`JoU?1Dx;|Fwf+_tR$v1zSaQk@YaQ71}y-yUq z+7T2{>*GLwvgg){KVl`?=?61dpIWetsS3iQ!rT1miDP}Ea&9f}sHV^Knr4({Uh!B@ zF+MxooPsBKjp?bdpX*rEtV9m4Wv9jX{KUw_8qGeQi}i()31oVZ%d1K!#utmv5+N+9 zU~x0PGqT9=&f+I4;N@$b=vc77v0xj)Gu8Si7~fo^M3H?}G9?WBx4Hv}y1$~{)9cJbCAi=2 zURetc_E7!b8F?~$;_<6tZU`_x>lI)ljhK_X}5!iJ~^UlJEx$ zbY{%T%nPN+_QTG=HSvtnt=yFM?5QAS?Tj18)r3_R#3sqiml4<`qP3( zzvrH$s@&RMF#p_{nJ%^M>Khb0!T8Il3m0_L5q|af>muFkuuesqya9h3`P>1g(!a93 zj(95B-+Qt|z_A}mtB(8rkHK8m2l1^@6vOeKQ#vpO5j--INcZqx!)aBu_E0^KzhL}( z4Fhgw%8Ex%g8844&b8x_v#MeLp%xndUE^*I;ap!+4E&Z|{C|(u8G9wIgX&GJ2X3d1 zo4b{{Rm2rnp z)Y;CO7*NjYnH*2t^V!S#XmGEDJIP2!C+(abN+IM)Cq-ugpR;;@PBwDagpB#7zTHX& z>+y6ZXE^y#q@J46)uY^c=aY!^r`UvvLhrS^9{5w1OiLH{FspXHQw>&PoF`XG_Ha6N zNyd(APewZV7a<&jdzyt`R+HS~rn@_C)oF{b(eO2M=PToMlUv1_Ppy3x`A)y^NKR#B z_w{C-p>d2vkujj1k27}<>v9S`eQ;o>o7milvrKh?u*y7hXEUi1o^>R&dhW0Cl}O%Vo^3eW zZ`A=U=hqDG*(cZP>~TI}oTD-DM^9=-c|6bCERvqHb2AP2L`#uhGF5i2QJE{U)A+R$ zK9J`gNNJ(R$l7M|)T(&r8Hwz$S`#|_VxG7A;K2^eudb>V`=#~63fURKLF9ociZWgcTyN7oCN3l(H_(b#SR^#YW97b7(YCL1fb4plX)rEb7?}fcc6}mz~gdhn0Oo!F;(~ZX_Z(2*%`M?CxK_;b(m_D_%X5{uMUK<&&#Q z@U6nlyyD2G^>t2G@Sc*&`AR*h_pM7e+!I``JX}y&P3LTNM=nVbd-1 zqoICWYg8vaCxmDGydJB;yLRVUooBnD=mh3ED`~7WKRY)V*Ih7Nqg}|wDkrQ?UvJYI z(ua|2biZRB{OfPRr|ao$ckb&AI?EaDWe?WtOj&QZmSqj2=d{Xrqr#h&oyxzEym@<< zno`}hELW85#*KlRYvBga2am-S{w4#@j@GTqsGn=c@6+(=y#Lq_PN?0qeDTTht`(JoI5l38tyF@ zT=T)!nNMGiw_4=0GI*-5TybvQoid|YO6YOxK&wn=cALRTgzM1?EdCAJZ(yyJNVEMb zcH1ex@XlaWuP6_ly4~P-57jzLB{JQYk|kcPvR}EIfU)UT??N!Aj0@WIhVq-#axPJF8uyzMe0#9WY#$mD4dQIB;k>ft^tj zR(wwDpn_&K(}2_cI(VSOv##;hLvdSA>0Mkkvp-{$VjTK+Uf~$;Zs~09GEgLNQagZ6 z_3Cg)=hp1X-StEL(1Gx(l&onl^I6xolHGOUf(0?BLC>|py<4d&chDd~)zesxhc%Yc z7^!E@oZP#bN4>#3y!f%3ERFKij~G>6W8 zn8K0v(hvQ(7qQnggLC(u&{^P#o2kHCOE5QT`4$Ux@uRP`;7{nwrV@i{YOpSo%`9|%I+i82aF`?Y;P1ur8Ul4tOpiW?!(JI z@M}t+9#k08wKikVw9c3`yde)B-1Mr9WT8t^<$TCs;*-8cT4#Or(m1YZzz6MbXH{RQ z@}UK5uT(~Z8hSTF|Meg|ys)u!ka~PeOWz-EX3PTnn1wz$+#B4`;omjsKX%g3 zL^z~Rsz_DL$2F#NR4+VcEgXF|k1wuEG)n7q^Bz2*@QqBr%#!)VRga!HrFKpj)PwH-*Phx#uSwUY3&Xc=OVw^>Mb))M9_9>Ho*5Q$vq#bU-eCl8hy?!%l%DSgK zbmD0}p?Ae_TA!Axe|q;|*Sg8-Z&2&hGbXINYpnC@nQzfE3kM_~WLciYvpSlzs}8Op z*{b2$#Uq){vu=e9&O^tZ(^RZ1wiS%?yr&4~=XTDz9waN>R4=#gc@q;v;*y&ww5yLQ zetxgm4DHnMDj#RPBQNM&Fjaq5LAy@;!k)M~Rd(vhODSJeoTP32+GDRdvF75wc(~!= z{hXyDBfGw&yXL;QO*gdrI2>L&aEV+dn|HV;FYC#y>K@Ek&AdZJFCTt0Gf}m(uI3fR z%}8qt&v=ZYu2*)S>XO-{0h(U{-H2Be1|GtIWaODriSg?Hmr=7ecCcPkSWfOdc1%}1 zNsH@iJ6}7=ucMkamAH+sYiw%IJ&pKNZSjemub*;yw|X*OP56ewtoZ7hd4-^|Hx8cG z)-ze9mNyMNXjn=*&vnN3Q)O@7l$W^KpLKWUEuBBRq_5W9FZtpx=eJJIiScNizj$w3 zI2vy!p5&pIZ=al8JqWp2K{aVC1n2&a#!Q53H)cE|*E<)v%(A+D?GVpky=!DM4o#zc zUaI`vn>xtL9~kiE@SfpdL}JNEGL3zz?`@nV-e}=!Kn*BUkB2%N?EgVDX}sbo5-$JgMxAI{Bf=Tioh4XYR9LeYmhvnSI<)QiXPp zXD~lf@}{~+y!Rh%m|0gUtNMEPxYZx)94l3*PGOZ*7!g7WmIJJZ{MVDRr}y z&p`>9R_A_>0f*Y>nnwoPOz;Ze2fGA)kPir;yIOiD9x`8&Gfc* zYMRA=mizj|adn{mW;aIYf1{+(+B!|veXq#%&BpUxAdyoz`?m&$49*SCs_1=t!a8%+ z^)J1;@|}{pW_Eh}u_wlNJA=ek7mmLDaWe6~H=I}nwH&jmvlZw2y%t<*K57u=2P2QQ zoOvBf$;A5M#HvWDum+X*cSZ@s{?VvqwF_W-mcahFVCRH5;Xs1;4#rOw46?X-o=(}G z!TD+97*o^Nak7Mrq`~}IXVQ70b0iEE|GZ?{%q4k@)Tv(--f}0eGfXR@=N z$#%wHb&hs&Keg+KcTn~Jy0O5gX2mitE5Z8Bly7#MRXh`)Wc2;FBkjyKE>9x&bsp;K z@788U9iG{6#rb{blsXn?o&1{f4S?+5>s8JE(SO2PitzK(IZWB zP%VER9PR0^_J|d)HAHa!QZk{KIQ*)sH94!l4nH`!;a#1xs=sx&behq+f4Y5tpHhKw zn$kaF&4>_%{gEOw_2fS# rrFSYjSxP6ye-{iR^qgI~&bDnkwr|;W%a(n1Y~Lv!`)=PkzW4e+$dzse literal 0 HcmV?d00001 diff --git a/test/functional/test_short_deck.py b/test/functional/test_short_deck.py index 889bc437..89cfd9b9 100644 --- a/test/functional/test_short_deck.py +++ b/test/functional/test_short_deck.py @@ -7,7 +7,7 @@ import numpy as np import dill as pickle -from pluribus.games.short_deck.state import ShortDeckPokerState +from pluribus.games.short_deck.state import ShortDeckPokerState, new_game from pluribus.games.short_deck.player import ShortDeckPokerPlayer from pluribus.poker.card import Card from pluribus.poker.pot import Pot @@ -338,3 +338,33 @@ def test_skips(n_players: int = 3): for i, action in enumerate(actions[fold_idx:]): if i % n_players == 0: assert action == "skip" + + +def test_load_game_state(n_players: int = 3, n: int = 2): + # Load a random sample of action sequences + random_actions_path = "research/size_of_problem/random_action_sequences.pkl" + action_sequences = _load_action_sequences(random_actions_path) + test_action_sequences = np.random.choice(action_sequences, n) + state: ShortDeckPokerState = new_game( + n_players, info_set_lut={}, real_time_test=True, public_cards=[] + ) + for action_sequence in test_action_sequences: + game_action_sequence = action_sequence.copy() + # Load current game state + current_game_state: ShortDeckPokerState = state.load_game_state( + offline_strategy={}, action_sequence=game_action_sequence + ) + current_history = current_game_state._history + check_action_seq_current = [] + for betting_stage in current_history.keys(): + check_action_seq_current += current_history[betting_stage] + check_action_sequence = [a for a in check_action_seq_current if a != "skip"] + assert check_action_sequence == action_sequence[:-1] + + new_state = current_game_state.deal_bayes() + full_history = new_state._history + check_action_seq_full = [] + for betting_stage in full_history.keys(): + check_action_seq_full += full_history[betting_stage] + check_action_sequence = [a for a in check_action_seq_full if a != "skip"] + assert check_action_sequence == action_sequence From 8d48cf361751a5fe84e13b2b0cd9c39a086ed617 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Tue, 2 Jun 2020 21:46:08 -0400 Subject: [PATCH 37/42] using smaller lookup table for pytest --- test/functional/test_short_deck.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test/functional/test_short_deck.py b/test/functional/test_short_deck.py index 89cfd9b9..89b02459 100644 --- a/test/functional/test_short_deck.py +++ b/test/functional/test_short_deck.py @@ -7,7 +7,8 @@ import numpy as np import dill as pickle -from pluribus.games.short_deck.state import ShortDeckPokerState, new_game +from pluribus.games.short_deck.state import ShortDeckPokerState, new_game, \ + InfoSetLookupTable from pluribus.games.short_deck.player import ShortDeckPokerPlayer from pluribus.poker.card import Card from pluribus.poker.pot import Pot @@ -345,8 +346,15 @@ def test_load_game_state(n_players: int = 3, n: int = 2): random_actions_path = "research/size_of_problem/random_action_sequences.pkl" action_sequences = _load_action_sequences(random_actions_path) test_action_sequences = np.random.choice(action_sequences, n) + # Lookup table that defaults to 0 as the cluster id + info_set_lut: InfoSetLookupTable = { + "pre_flop": collections.defaultdict(lambda: 0), + "flop": collections.defaultdict(lambda: 0), + "turn": collections.defaultdict(lambda: 0), + "river": collections.defaultdict(lambda: 0), + } state: ShortDeckPokerState = new_game( - n_players, info_set_lut={}, real_time_test=True, public_cards=[] + n_players, info_set_lut=info_set_lut, real_time_test=True, public_cards=[] ) for action_sequence in test_action_sequences: game_action_sequence = action_sequence.copy() From 8e055333e74fda13791eeac25b41b2219d5e3d45 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Mon, 8 Jun 2020 23:48:41 -0400 Subject: [PATCH 38/42] regresion-style test for loading public info, won't work until I add a file for this, it is also lengthy at the moment --- test/functional/test_short_deck.py | 85 ++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/test/functional/test_short_deck.py b/test/functional/test_short_deck.py index 89b02459..4d31cb8d 100644 --- a/test/functional/test_short_deck.py +++ b/test/functional/test_short_deck.py @@ -1,7 +1,9 @@ import collections +import json import copy import random from typing import List, Tuple, Optional +import joblib import pytest import numpy as np @@ -13,6 +15,7 @@ from pluribus.poker.card import Card from pluribus.poker.pot import Pot from pluribus.utils.random import seed +from pluribus.poker.deck import Deck def _new_game( @@ -376,3 +379,85 @@ def test_load_game_state(n_players: int = 3, n: int = 2): check_action_seq_full += full_history[betting_stage] check_action_sequence = [a for a in check_action_seq_full if a != "skip"] assert check_action_sequence == action_sequence + + +def test_public_cards(n_players: int = 3, n: int = 5): + # TODO: Combine this with above test if possible.. + # TODO: Move any needed files within the test folder and condense.. + strategy_dir = "research/test_methodology/test_strategy2/" + strategy_path = "unnormalized_output/offline_strategy_1500.gz" + check = joblib.load(strategy_dir + strategy_path) + histories = np.random.choice(list(check.keys()), n) + action_sequences = [] + public_cards_lst = [] + community_card_dict = { + "pre_flop": 0, + "flop": 3, + "turn": 4, + "river": 5, + } + ranks = list(range(10, 14 + 1)) + deck = Deck(include_ranks=ranks) + for history in histories: + history_dict = json.loads(history) + history_lst = history_dict["history"] + action_sequence = [] + betting_rounds = [] + for x in history_lst: + action_sequence += list(x.values())[0] + betting_rounds += list(x.keys()) + action_sequences.append(action_sequence) + final_betting_round = list(betting_rounds)[-1] + n_cards = community_card_dict[final_betting_round] + cards_in_deck = deck._cards_in_deck + public_cards = np.random.choice(cards_in_deck, n_cards, replace=False) + public_cards_lst.append(list(public_cards)) + + info_set_lut: InfoSetLookupTable = { + "pre_flop": collections.defaultdict(lambda: 0), + "flop": collections.defaultdict(lambda: 0), + "turn": collections.defaultdict(lambda: 0), + "river": collections.defaultdict(lambda: 0), + } + for i in range(0, len(action_sequences)): + public_cards = public_cards_lst[i].copy() + if not public_cards: + continue + action_sequence = action_sequences[i].copy() + state: ShortDeckPokerState = new_game( + n_players, + info_set_lut=info_set_lut, + real_time_test=True, + public_cards=public_cards, + ) + current_game_state: ShortDeckPokerState = state.load_game_state( + offline_strategy={}, action_sequence=action_sequence + ) + new_state = current_game_state.deal_bayes() + + cont = True + if len(public_cards) == 0: + loaded_betting_stage = "pre_flop" + elif len(public_cards) == 3: + loaded_betting_stage = "flop" + elif len(public_cards) == 4: + loaded_betting_stage = "turn" + elif len(public_cards) == 5: + loaded_betting_stage = "river" + + public_info = new_state._public_information + for betting_stage in public_info.keys(): + if betting_stage == "pre_flop": + # No cards in the pre_flop stage.. + continue + if cont: + card_len = community_card_dict[betting_stage] + assert public_cards[:card_len] == public_info[betting_stage] + if betting_stage == loaded_betting_stage: + cont = False + else: + # Should only get here if we hit the last action_sequence of + # a round.. + state_public_card_len = len(new_state.community_cards) + public_card_len = len(public_cards) + assert state_public_card_len == public_card_len + 1 From cc7393377d250e2b112f11434696b2e871f44b1b Mon Sep 17 00:00:00 2001 From: big-c-note Date: Sat, 13 Jun 2020 09:10:00 -0400 Subject: [PATCH 39/42] reorganizing some --- research/{test_methodology => rts}/RT.py | 0 research/rts/RT_cfr.py | 181 ++++++++++++++ .../average_unnormalized_strategy.py | 91 +++++++ research/stat_tests/bot_test.py | 152 ++++++++++++ research/stat_tests/entry_rts_ab.py | 67 ++++++ ...ng_nash_equilibriums_via_simulations.ipynb | 224 ++++++++++++++++++ test/regression/check_bayes.py | 157 ++++++++++++ 7 files changed, 872 insertions(+) rename research/{test_methodology => rts}/RT.py (100%) create mode 100644 research/rts/RT_cfr.py create mode 100644 research/stat_tests/average_unnormalized_strategy.py create mode 100644 research/stat_tests/bot_test.py create mode 100644 research/stat_tests/entry_rts_ab.py create mode 100644 research/stat_tests/validating_nash_equilibriums_via_simulations.ipynb create mode 100644 test/regression/check_bayes.py diff --git a/research/test_methodology/RT.py b/research/rts/RT.py similarity index 100% rename from research/test_methodology/RT.py rename to research/rts/RT.py diff --git a/research/rts/RT_cfr.py b/research/rts/RT_cfr.py new file mode 100644 index 00000000..4cf69d48 --- /dev/null +++ b/research/rts/RT_cfr.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +import collections +from typing import Dict +import joblib +from pathlib import Path + +from tqdm import trange +import numpy as np +import datetime +import yaml + +from pluribus import utils +from pluribus.games.short_deck.state import ShortDeckPokerState, new_game +from pluribus.games.short_deck.agent import Agent + + +def normalize_strategy(this_info_sets_regret: Dict[str, float]) -> Dict[str, float]: + """Calculate the strategy based on the current information sets regret.""" + actions = this_info_sets_regret.keys() + regret_sum = sum([max(regret, 0) for regret in this_info_sets_regret.values()]) + if regret_sum > 0: + strategy: Dict[str, float] = { + action: max(this_info_sets_regret[action], 0) / regret_sum + for action in actions + } + elif this_info_sets_regret == {}: + # Don't return strategy if no strategy was made + # during training + strategy: Dict[str, float] = {} + elif regret_sum == 0: + # Regret is negative, we learned something + default_probability = 1 / len(actions) + strategy: Dict[str, float] = {action: default_probability for action in actions} + return strategy + + +def calculate_strategy( + regret: Dict[str, Dict[str, float]], + I: str, + state: ShortDeckPokerState, +) -> Dict[str, Dict[str, float]]: + """ + Calculate strategy based on regret + """ + sigma = collections.defaultdict(lambda: collections.defaultdict(lambda: 1 / 3)) + rsum = sum([max(x, 0) for x in regret[I].values()]) + for a in state.legal_actions: + if rsum > 0: + sigma[I][a] = max(regret[I][a], 0) / rsum + else: + sigma[I][a] = 1 / len(state.legal_actions) + return sigma + + +def _create_dir(folder_id: str) -> Path: + """Create and get a unique dir path to save to using a timestamp.""" + time = str(datetime.datetime.now()) + for char in ":- .": + time = time.replace(char, "_") + path: Path = Path(f"./{folder_id}_results_{time}") + path.mkdir(parents=True, exist_ok=True) + return path + + +def cfr(agent: Agent, state: ShortDeckPokerState, i: int, t: int) -> float: + """ + CFR algo with the a temporary regret object for better strategy averaging + """ + ph = state.player_i + + player_not_in_hand = not state.players[i].is_active + if state.is_terminal or player_not_in_hand: + return state.payout[i] + + elif ph == i: + I = state.info_set + # Move regret over to temporary object and build off that + if agent.tmp_regret[I] == {}: + agent.tmp_regret[I] == agent.regret[I].copy() + sigma = calculate_strategy(agent.tmp_regret, I, state) + + vo = 0.0 + voa = {} + for a in state.legal_actions: + new_state: ShortDeckPokerState = state.apply_action(a) + voa[a] = cfr(agent, new_state, i, t) + vo += sigma[I][a] * voa[a] + + for a in state.legal_actions: + agent.tmp_regret[I][a] += voa[a] - vo + + return vo + else: + Iph = state.info_set + # Move regret over to a temporary object and build off that + if agent.tmp_regret[Iph] == {}: + agent.tmp_regret[Iph] == agent.regret[Iph].copy() + sigma = calculate_strategy(agent.regret, Iph, state) + + try: + a = np.random.choice( + list(sigma[Iph].keys()), 1, p=list(sigma[Iph].values()), + )[0] + + except KeyError: + p = 1 / len(state.legal_actions) + probabilities = np.full(len(state.legal_actions), p) + a = np.random.choice(state.legal_actions, p=probabilities) + sigma[Iph] = {action: p for action in state.legal_actions} + + new_state: ShortDeckPokerState = state.apply_action(a) + return cfr(agent, new_state, i, t) + + +def rts( + offline_strategy_path: str, + regret_path: str, + public_cards: list, + action_sequence: list, + n_iterations: int, + lcfr_threshold: int, + discount_interval: int, + n_players: int, + update_interval: int, + update_threshold: int, + dump_int: int, +): + """RTS.""" + config: Dict[str, int] = {**locals()} + save_path: Path = _create_dir('RTS') + with open(save_path / "config.yaml", "w") as steam: + yaml.dump(config, steam) + # TODO: fix the seed + # utils.random.seed(36) + agent = Agent(regret_path=regret_path) + # Load unnormalized strategy to build off + offline_strategy = joblib.load(offline_strategy_path) + state: ShortDeckPokerState = new_game( + 3, real_time_test=True, public_cards=public_cards + ) + # Load current game state + current_game_state: ShortDeckPokerState = state.load_game_state( + offline_strategy, action_sequence + ) + # We don't need the offline strategy for search.. + # del offline_strategy + for t in trange(1, n_iterations + 1, desc="train iter"): + for i in range(n_players): # fixed position i + # Deal hole cards based on bayesian updating of hole card probs + state: ShortDeckPokerState = current_game_state.deal_bayes() + cfr(agent, state, i, t) + if t < lcfr_threshold & t % discount_interval == 0: + d = (t / discount_interval) / ((t / discount_interval) + 1) + for I in agent.tmp_regret.keys(): + for a in agent.tmp_regret[I].keys(): + agent.tmp_regret[I][a] *= d + # Add the unnormalized strategy into the original + # Right now assumes dump_int is a multiple of n_iterations + if t % dump_int == 0: + # offline_strategy = joblib.load(offline_strategy_path) + # Adding the regret back to the regret dict, we'll build off for next RTS + for I in agent.tmp_regret.keys(): + if agent.tmp_regret != {}: + agent.regret[I] = agent.tmp_regret[I].copy() + for info_set, this_info_sets_regret in sorted(agent.tmp_regret.items()): + # If this_info_sets_regret == {}, we do nothing + strategy = normalize_strategy(this_info_sets_regret) + # Check if info_set exists.. + no_info_set = info_set not in offline_strategy + if no_info_set or offline_strategy[info_set] == {}: + offline_strategy[info_set] = {a: 0 for a in strategy.keys()} + for action, probability in strategy.items(): + try: + offline_strategy[info_set][action] += probability + except: + import ipdb; + ipdb.set_trace() + agent.reset_new_regret() + + return agent, offline_strategy diff --git a/research/stat_tests/average_unnormalized_strategy.py b/research/stat_tests/average_unnormalized_strategy.py new file mode 100644 index 00000000..e35965ad --- /dev/null +++ b/research/stat_tests/average_unnormalized_strategy.py @@ -0,0 +1,91 @@ +import collections +import glob +import os +import re +from typing import Dict, List, Union + +import click +import joblib +from tqdm import tqdm + + +def calculate_strategy(this_info_sets_regret: Dict[str, float]) -> Dict[str, float]: + """Calculate the strategy based on the current information sets regret.""" + actions = this_info_sets_regret.keys() + regret_sum = sum([max(regret, 0) for regret in this_info_sets_regret.values()]) + if regret_sum > 0: + strategy: Dict[str, float] = { + action: max(this_info_sets_regret[action], 0) / regret_sum + for action in actions + } + elif this_info_sets_regret == {}: + # Don't return strategy if no strategy was made + # during training + strategy: Dict[str, float] = {} + elif regret_sum == 0: + # Regret is negative, we learned something + default_probability = 1 / len(actions) + strategy: Dict[str, float] = {action: default_probability for action in actions} + return strategy + + +def try_to_int(text: str) -> Union[str, int]: + """Attempt to return int.""" + return int(text) if text.isdigit() else text + + +def natural_key(text): + """Sort with natural numbers.""" + return [try_to_int(c) for c in re.split(r"(\d+)", text)] + + +def average_strategy(all_file_paths: List[str]) -> Dict[str, Dict[str, float]]: + """Compute the mean strategy over all timesteps.""" + # The offline strategy for all information sets. + offline_strategy: Dict[str, Dict[str, float]] = collections.defaultdict( + lambda: collections.defaultdict(lambda: 0.0) + ) + # Sum up all strategies. + for dump_path in tqdm(all_file_paths, desc="loading dumps"): + # Load file. + try: + agent = joblib.load(dump_path) + except Exception as e: + tqdm.write(f"Failed to load file at {dump_path} because:{e}") + agent = {} + regret = agent.get("regret", {}) + # Sum probabilities from computed strategy.. + for info_set, this_info_sets_regret in sorted(regret.items()): + strategy = calculate_strategy(this_info_sets_regret) + # If strategy == {}, we do nothing + for action, probability in strategy.items(): + offline_strategy[info_set][action] += probability + # Return regular dict, not defaultdict. + return {info_set: dict(strategy) for info_set, strategy in offline_strategy.items()} + + +@click.command() +@click.option( + "--results_dir_path", default=".", help="the location of the agent file dumps." +) +@click.option( + "--write_dir_path", default=".", help="where to save the offline strategy" +) +def cli(results_dir_path: str, write_dir_path: str): + """Compute the strategy and write to file.""" + # Find all files to load. + all_file_paths = glob.glob(os.path.join(results_dir_path, "*.gz")) + if not all_file_paths: + raise ValueError(f"No agent dumps could be found at: {results_dir_path}") + # Sort the file paths in the order they were created. + all_file_paths = sorted(all_file_paths, key=natural_key) + offline_strategy = average_strategy(all_file_paths) + # Save dictionary to compressed file. + latest_file = os.path.basename(all_file_paths[-1]) + latest_iteration: int = int(re.findall(r"\d+", latest_file)[0]) + save_file: str = f"offline_strategy_{latest_iteration}.gz" + joblib.dump(offline_strategy, os.path.join(write_dir_path, save_file)) + + +if __name__ == "__main__": + cli() diff --git a/research/stat_tests/bot_test.py b/research/stat_tests/bot_test.py new file mode 100644 index 00000000..678470e4 --- /dev/null +++ b/research/stat_tests/bot_test.py @@ -0,0 +1,152 @@ +from typing import List, Dict, DefaultDict +from pathlib import Path +import joblib +import collections + +from tqdm import trange +import yaml +import datetime +import numpy as np +from scipy import stats + +from pluribus.games.short_deck.state import ShortDeckPokerState, new_game +from pluribus.poker.card import Card + +def _calculate_strategy( + state: ShortDeckPokerState, + I: str, + strategy: DefaultDict[str, DefaultDict[str, float]], + count=None, + total_count=None +) -> str: + sigma = collections.defaultdict(lambda: collections.defaultdict(lambda: 1 / 3)) + try: + # If strategy is empty, go to other block + sigma[I] = strategy[I].copy() + if sigma[I] == {}: + raise KeyError + norm = sum(sigma[I].values()) + for a in sigma[I].keys(): + sigma[I][a] /= norm + a = np.random.choice( + list(sigma[I].keys()), 1, p=list(sigma[I].values()), + )[0] + except KeyError: + if count is not None: + count += 1 + p = 1 / len(state.legal_actions) + probabilities = np.full(len(state.legal_actions), p) + a = np.random.choice(state.legal_actions, p=probabilities) + sigma[I] = {action: p for action in state.legal_actions} + if total_count is not None: + total_count += 1 + return a, count, total_count + + +def _create_dir(folder_id: str) -> Path: + """Create and get a unique dir path to save to using a timestamp.""" + time = str(datetime.datetime.now()) + for char in ":- .": + time = time.replace(char, "_") + path: Path = Path(f"./{folder_id}_results_{time}") + path.mkdir(parents=True, exist_ok=True) + return path + + +def agent_test( + hero_strategy_path: str, + opponent_strategy_path: str, + real_time_est: bool = False, + action_sequence: List[str] = None, + public_cards: List[Card] = [], + n_outter_iters: int = 30, + n_inner_iters: int = 100, + n_players: int = 3, + hero_count=None, + hero_total_count=None, +): + config: Dict[str, int] = {**locals()} + save_path: Path = _create_dir('bt') + with open(save_path / "config.yaml", "w") as steam: + yaml.dump(config, steam) + + # Load unnormalized strategy for hero + hero_strategy = joblib.load(hero_strategy_path) + # Load unnormalized strategy for opponents + opponent_strategy = joblib.load(opponent_strategy_path) + + # Loading game state we used RTS on + if real_time_est: + state: ShortDeckPokerState = new_game( + n_players, real_time_test=real_time_est, public_cards=public_cards + ) + current_game_state: ShortDeckPokerState = state.load_game_state( + opponent_strategy, action_sequence + ) + + # TODO: Right now, this can only be used for loading states if the two strategies + # are averaged. Even averaging strategies is risky. Loading a game state + # should be used with caution. It will work only if the probability of reach + # is identical across strategies. Use the average strategy. + + info_set_lut = {} + EVs = np.array([]) + for _ in trange(1, n_outter_iters): + EV = np.array([]) # Expected value for player 0 (hero) + for t in trange(1, n_inner_iters + 1, desc="train iter"): + for p_i in range(n_players): + if real_time_est: + # Deal hole cards based on bayesian updating of hole card probs + state: ShortDeckPokerState = current_game_state.deal_bayes() + else: + state: ShortDeckPokerState = new_game(n_players, info_set_lut) + info_set_lut = state.info_set_lut + while True: + player_not_in_hand = not state.players[p_i].is_active + if state.is_terminal or player_not_in_hand: + EV = np.append(EV, state.payout[p_i]) + break + if state.player_i == p_i: + random_action, hero_count, hero_total_count = _calculate_strategy( + state, + state.info_set, + hero_strategy, + count=hero_count, + total_count=hero_total_count + ) + else: + random_action, oc, otc = _calculate_strategy( + state, + state.info_set, + opponent_strategy, + ) + state = state.apply_action(random_action) + EVs = np.append(EVs, EV.mean()) + t_stat = (EVs.mean() - 0) / (EVs.std() / np.sqrt(n_outter_iters)) + p_val = stats.t.sf(np.abs(t_stat), n_outter_iters - 1) + results_dict = { + 'Expected Value': float(EVs.mean()), + 'T Statistic': float(t_stat), + 'P Value': float(p_val), + 'Standard Deviation': float(EVs.std()), + 'N': int(len(EVs)), + 'Random Moves Hero': hero_count, + 'Total Moves Hero': hero_total_count + } + with open(save_path / 'results.yaml', "w") as stream: + yaml.safe_dump(results_dict, stream=stream, default_flow_style=False) + + +if __name__ == "__main__": + strat_path = "test_strategy2/unnormalized_output/" + agent_test( + hero_strategy_path=strat_path + "random_strategy.gz", + opponent_strategy_path=strat_path + "offline_strategy_1500.gz", + real_time_est=False, + public_cards=[], + action_sequence=None, + n_inner_iters=25, + n_outter_iters=75, + hero_count=0, + hero_total_count=0 + ) diff --git a/research/stat_tests/entry_rts_ab.py b/research/stat_tests/entry_rts_ab.py new file mode 100644 index 00000000..df29136e --- /dev/null +++ b/research/stat_tests/entry_rts_ab.py @@ -0,0 +1,67 @@ +import numpy as np +import json +import joblib + +from RT_cfr import rts +from bot_test import agent_test +from pluribus.poker.deck import Deck + + +check = joblib.load('test_strategy2/unnormalized_output/offline_strategy_1500.gz') +histories = np.random.choice(list(check.keys()), 2) +action_sequences = [] +public_cards_lst = [] +community_card_dict = { + "pre_flop": 0, + "flop": 3, + "turn": 4, + "river": 5, +} +ranks = list(range(10, 14 + 1)) +deck = Deck(include_ranks=ranks) +for history in histories: + history_dict = json.loads(history) + history_lst = history_dict['history'] + action_sequence = [] + betting_rounds = [] + for x in history_lst: + action_sequence += list(x.values())[0] + betting_rounds += list(x.keys()) + action_sequences.append(action_sequence) + final_betting_round = list(betting_rounds)[-1] + n_cards = community_card_dict[final_betting_round] + cards_in_deck = deck._cards_in_deck + public_cards = np.random.choice(cards_in_deck, n_cards) + public_cards_lst.append(list(public_cards)) + +for i in range(0, len(action_sequences)): +# try: + public_cards = public_cards_lst[i].copy() + action_sequence = action_sequences[i].copy() + agent_output, offline_strategy = rts( + 'test_strategy2/unnormalized_output/offline_strategy_1500.gz', + 'test_strategy2/strategy_1500.gz', public_cards, action_sequence, + 1500, 400, 400, 3, 400, 400, 20 + ) + save_path = "test_strategy2/unnormalized_output/" + last_regret = {info_set: dict(strategy) for info_set, strategy in agent_output.regret.items()} + joblib.dump(offline_strategy, save_path + f'rts_output{i+35}.gz', compress="gzip") + joblib.dump(last_regret, save_path + f'last_regret{i+35}.gz', compress="gzip") + + public_cards = public_cards_lst[i].copy() + action_sequence = action_sequences[i].copy() + strat_path = "test_strategy2/unnormalized_output/" + agent_test( + hero_strategy_path=strat_path + f"rts_output{i+35}.gz", + opponent_strategy_path=strat_path + "offline_strategy_1500.gz", + real_time_est=True, + public_cards=public_cards, + action_sequence=action_sequence, + n_inner_iters=25, + n_outter_iters=250, + hero_count=0, + hero_total_count=0, + ) +# except: +# print(f"ERROR on {action_sequences[i]}, {public_cards_lst[i]}") +# pass diff --git a/research/stat_tests/validating_nash_equilibriums_via_simulations.ipynb b/research/stat_tests/validating_nash_equilibriums_via_simulations.ipynb new file mode 100644 index 00000000..c146c559 --- /dev/null +++ b/research/stat_tests/validating_nash_equilibriums_via_simulations.ipynb @@ -0,0 +1,224 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Validating Nash Equilibriums Via Simulations" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "_by Colin Manko_" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In an effort to validate and test possible improvements to core poker artificial intelligence algorithms, I have designed the following methodology." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Goals" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- Validate that MCCFR offline learning strategy is approximating a Nash equilibrium\n", + "- More generally, create a methodology that allows for rigorously testing changes made to the core AI algorithms" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Prerequisites" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- Need two test bot implementation strategies ($\\beta{1}$ and $\\beta{2}$) that we would like to compare\n", + "- Need a human tester ($H_{0}$) as a quasi control. The human tester should not have access to any underlying strategies from the test bots or simulated Nash" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Step 1: Randomly Generate Test Game" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Given a set of $N$ game tree nodes (this is the entire game tree, as given by infoset, $I$), randomly generate $x$ test nodes without preplacement and with equal probability. Call the set of test nodes $U$.\n", + "\n", + "As a side note, we will account for probability of reach ($p(h)$) in another step. Equal probability across nodes allows us to find patterns across nodes where our agent underperforms. We will adjust the expected value at $I$, ($v^{\\sigma}(I)$), by $p(h)$.\n", + "- **How to**: For Limit Texas Hold'em, the number of action sequences ($N$), is small enough that they can be found computationally rather than analytically. We can run *all_action_sequences.py* in the *size_of_problem* directory to generate this list. \n", + "- _Something like 15-20 hours and less than 4GB??_\n", + "- Generate $x$ integers to be indices and select them from the *all_action_sequences.py* output\n", + "- Once $x$ action sequences are generated, randomly generate $x$ public card combos, based on the betting stage of the test node, $u$, as well as one pair of private hole cards to be used by $\\beta{1}$, $\\beta{2}$ and $H_0$. They will only get that hand at $u$. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Step 2: Prepare Realtime Search for Finding Nash Equilibrium" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For each test node, $u$, in $U$, use realtime search to compute the Nash Equilibrium ($\\sigma^*$) by constraining the search algorithm to start at $u$, where $u$ is equivalent to $I$ in regard to action sequence, but does not have any set hand for the traversing player ($p_i$).\n", + "\n", + "Use a pooled strategy between $\\beta{1}$ and $\\beta{2}$ to estimate $p(h)$ without bias:" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For hand in possible combinations of real hands:\n", + "
\n", + "    For idx, $a$ in action sequence at $u$:\n", + "
\n", + "        if idx == 0:\n", + "
\n", + "             $p(h)_\\beta{1}$ = $\\beta{1}$[$I$][$a$]\n", + "
\n", + "             $p(h)_\\beta{2}$ = $\\beta{2}$[$I$][$a$]\n", + "
\n", + "        $p(h)_\\beta{1}$ *= $\\beta{1}$[$I$][$a$]\n", + "
\n", + "        $p(h)_\\beta{2}$ *= $\\beta{1}$[$I$][$a$]\n", + "
\n", + "    p(h)[rs] = ($p(h)_\\beta{1}$ + $p(h)_\\beta{2}$)/2" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The root node of the realtime search algorithm is replaced with a chance node that represents each possible node in the public state $G$ [[Brown, Sandholm, Amos]](https://papers.nips.cc/paper/7993-depth-limited-solving-for-imperfect-information-games.pdf). From the above psuedo-code, this deal can be generated as: " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Generate deal order\n", + "
\n", + "For $i$ in $P_i$ deal order:\n", + "
\n", + "    Generate hand for player based on normalized $p(h)[rs]$ if available, else try again" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "_The \"if available, else try again\" part could be made better_" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Other important features of the _\"Nash Bot\"_ real time search..**:\n", + "- The _\"Nash bot\"_ is the master of this node. In order to reach full convergence, from the normal MCCFR algorithm, we must remove the sampling of actions for opponents.\n", + "- For ease, the real time search should not use leaf nodes, but should search to the end of the game tree, where either a terminal node or a shown down is entered. In this way, we can get a truer sense of the expected value." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "A Nash Equilibrium is found if the change in strategy on each iteration drops below some threshold $t$ for the real hand we are testing for. Charting probabilities for each action in $u$ over time for the randomly generated real hand to test should show a convergence over time." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "_One main benefit of using this real time search to validate CFR is this search will need to be developed anyway._" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Step 3: Test and Measure Success" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**For the test bots:**\n", + "For each $u$ in $U$, play each test strategy ($\\beta{1}$ and $\\beta{2}$) against the _\"Nash bot\"_ for $r$ number of simulations. The _\"Nash bot\"_ should be dealt available hands from the distribution of probabilities as determined by $p(h)[rs]$ in the pseudo-code above. Both the test bots and the human tester will be dealt the same hand in each simulation of game play on $u$, as randomly generated in step 1. \n", + "\n", + "If $\\beta{1}$ or $\\beta{2}$ has converged to a Nash equilibrium, then we should expect $v^\\sigma$ to be equal to 0 for our test bot, assuming that _\"Nash bot\"_ has converged to a Nash equilibrium itself. $v^{\\sigma^*}(u)$ and $v^{\\sigma}(u)$ are the estimated payouts for the _\"Nash bot\"_ opponents and the \"hero\" (test bots or human), respectively.\n", + "\n", + "**For the human tester:**\n", + "We can simply create a contrived game. Based on the normalized probability of reach for $u$, $\\bar{p(h)}$, we can randomly generate which $u$ the human player is entered into, however they will always have the same hand upon entering $u$ and their opponents hands will vary based on $p(h)[rs]$.\n", + "\n", + "The test metric is as follows, after $p(h)$ has been normalized for space $U$, $\\bar{p(h)}$:\n", + "$$\\sum_{i=1}^{x}(v^{\\sigma}(u_i)-v^{\\sigma^*}(u_{-i}))\\times{\\bar{p(h)}}$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The value closest to 0 (summing no testing agent goes over 0) will have best approximated the Nash equilibrium. Additionaly, $H_0$ can be used as a quasi-control, to validate that the bot is beating a human.\n", + "\n", + "The above metric also has some degree of simulation error. For each simulation in $r$ simulations, we create a distribution of values that has a standard deviation and follows the normal distribution. \n", + "\n", + "Along with calculating the expected payout per simulation, $u^{\\sigma}(u_i)-u^{\\sigma^*}(u_{-i})$, we can also calculate $\\sigma$ for this distribution in order to describe a confidence interval around the test metric. \n", + "\n", + "Finally, a simple difference of means can be done between each test bot to decipher a winner and if that winner had a statistically significant edge. We can then study each $u$ in $U$ to find patterns in which nodes the espspective bots did not do well with." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.0" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/test/regression/check_bayes.py b/test/regression/check_bayes.py new file mode 100644 index 00000000..d42faf5f --- /dev/null +++ b/test/regression/check_bayes.py @@ -0,0 +1,157 @@ +import joblib +import collections +import json +from typing import DefaultDict + +import numpy as np +from tqdm import trange + +from pluribus.poker.deck import Deck +from pluribus.games.short_deck.state import ShortDeckPokerState, new_game + + +def _calculate_strategy( + state: ShortDeckPokerState, + I: str, + strategy: DefaultDict[str, DefaultDict[str, float]], +) -> str: + sigma = collections.defaultdict(lambda: collections.defaultdict(lambda: 1 / 3)) + try: + # If strategy is empty, go to other block + sigma[I] = strategy[I].copy() + if sigma[I] == {}: + raise KeyError + norm = sum(sigma[I].values()) + for a in sigma[I].keys(): + sigma[I][a] /= norm + a = np.random.choice( + list(sigma[I].keys()), 1, p=list(sigma[I].values()), + )[0] + except KeyError: + p = 1 / len(state.legal_actions) + probabilities = np.full(len(state.legal_actions), p) + a = np.random.choice(state.legal_actions, p=probabilities) + sigma[I] = {action: p for action in state.legal_actions} + return a + + +n = 10000 +n_players = 3 +inner_iters = 1000 + +strategy_dir = "research/test_methodology/test_strategy2/" +strategy_path = "unnormalized_output/offline_strategy_1500.gz" +check = joblib.load(strategy_dir + strategy_path) +histories = np.random.choice(list(check.keys()), n) +action_sequences = [] +public_cards_lst = [] +community_card_dict = { + "pre_flop": 0, + "flop": 3, + "turn": 4, + "river": 5, +} +# Shorter deck for more reasonable simulation time.. +ranks = list(range(12, 14 + 1)) +deck = Deck(include_ranks=ranks) +found = 0 +for idx, history in enumerate(histories): + if idx % 100 == 0: + print(idx) + history_dict = json.loads(history) + history_lst = history_dict["history"] + if history_lst == []: + continue + action_sequence = [] + betting_rounds = [] + for x in history_lst: + action_sequence += list(x.values())[0] + betting_rounds += list(x.keys()) + try: + final_betting_round = list(betting_rounds)[-1] + except: + import ipdb; + ipdb.set_trace() + # Hacking this for now, keeping the simulation small.. + if len(action_sequence) > 2: + continue + action_sequences.append(action_sequence) + n_cards = community_card_dict[final_betting_round] + cards_in_deck = deck._cards_in_deck + public_cards = np.random.choice(cards_in_deck, n_cards, replace=False) + public_cards_lst.append(list(public_cards)) + found += 1 + if found == 2: + break + # Assuming we find 2 action sequences a=out of 1000 + +store_hand_probs = {} +for i in trange(0, len(action_sequences)): + public_cards = public_cards_lst[i].copy() + # will need to check for this bug later.. +# if not public_cards: +# import ipdb; +# ipdb.set_trace() + action_sequence = action_sequences[i].copy() + state: ShortDeckPokerState = new_game( + n_players, + real_time_test=True, + public_cards=public_cards, + ) + current_game_state: ShortDeckPokerState = state.load_game_state( + offline_strategy=check, action_sequence=action_sequence + ) + new_state = current_game_state.deal_bayes() + + this_hand_probs = new_state._starting_hand_probs.copy() + for p_i in this_hand_probs.keys(): + for starting_hand in this_hand_probs[p_i].keys(): + x = this_hand_probs[p_i][starting_hand] + this_hand_probs[p_i][starting_hand] = {'deal_bayes':x, 'sim':None} + + action_sequence = action_sequences[i].copy() + public_cards = public_cards_lst[i].copy() + info_set_lut = {} + cont = True + actions = [] + tries = 0 + success = 0 + hand_dict = {0: {}, 1: {}, 2: {}} + while cont: + state: ShortDeckPokerState = new_game( + n_players, + info_set_lut, + real_time_test=True, + public_cards=public_cards + ) + info_set_lut = state.info_set_lut + while True: + count = 0 + if tries == 1000: # definitely a hack need to be careful about this + # value + for p_i in state.players: + hole_cards = tuple(state.players[p_i].cards) + try: + hand_dict[p_i][hole_cards] += 0 + except KeyError: + hand_dict[p_i][hole_cards] = 0 + random_action = _calculate_strategy(state, state.info_set, check) + if random_action != action_sequence[count]: + tries += 1 + break + new_state = state.apply_action(random_action) + actions.append(random_action) + if actions == action_sequence: + for p_i in state.players: + hole_cards = tuple(state.players[p_i].cards) + try: + hand_dict[p_i][hole_cards] += 1 + except KeyError: + hand_dict[p_i][hole_cards] = 1 + success += 1 + break + count += 1 + if success == 1: + break + import ipdb; + ipdb.set_trace() From 1eb9a85ed3772809b8fce4cf2a65831eea9524a3 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Sat, 13 Jun 2020 09:18:38 -0400 Subject: [PATCH 40/42] removing old files --- research/test_methodology/RT_cfr.py | 181 -------------- .../average_unnormalized_strategy.py | 91 ------- research/test_methodology/bot_test.py | 152 ------------ research/test_methodology/entry_rts_ab.py | 67 ------ ...ng_nash_equilibriums_via_simulations.ipynb | 224 ------------------ research/to_do.md | 49 ---- 6 files changed, 764 deletions(-) delete mode 100644 research/test_methodology/RT_cfr.py delete mode 100644 research/test_methodology/average_unnormalized_strategy.py delete mode 100644 research/test_methodology/bot_test.py delete mode 100644 research/test_methodology/entry_rts_ab.py delete mode 100644 research/test_methodology/validating_nash_equilibriums_via_simulations.ipynb delete mode 100644 research/to_do.md diff --git a/research/test_methodology/RT_cfr.py b/research/test_methodology/RT_cfr.py deleted file mode 100644 index 4cf69d48..00000000 --- a/research/test_methodology/RT_cfr.py +++ /dev/null @@ -1,181 +0,0 @@ -from __future__ import annotations - -import collections -from typing import Dict -import joblib -from pathlib import Path - -from tqdm import trange -import numpy as np -import datetime -import yaml - -from pluribus import utils -from pluribus.games.short_deck.state import ShortDeckPokerState, new_game -from pluribus.games.short_deck.agent import Agent - - -def normalize_strategy(this_info_sets_regret: Dict[str, float]) -> Dict[str, float]: - """Calculate the strategy based on the current information sets regret.""" - actions = this_info_sets_regret.keys() - regret_sum = sum([max(regret, 0) for regret in this_info_sets_regret.values()]) - if regret_sum > 0: - strategy: Dict[str, float] = { - action: max(this_info_sets_regret[action], 0) / regret_sum - for action in actions - } - elif this_info_sets_regret == {}: - # Don't return strategy if no strategy was made - # during training - strategy: Dict[str, float] = {} - elif regret_sum == 0: - # Regret is negative, we learned something - default_probability = 1 / len(actions) - strategy: Dict[str, float] = {action: default_probability for action in actions} - return strategy - - -def calculate_strategy( - regret: Dict[str, Dict[str, float]], - I: str, - state: ShortDeckPokerState, -) -> Dict[str, Dict[str, float]]: - """ - Calculate strategy based on regret - """ - sigma = collections.defaultdict(lambda: collections.defaultdict(lambda: 1 / 3)) - rsum = sum([max(x, 0) for x in regret[I].values()]) - for a in state.legal_actions: - if rsum > 0: - sigma[I][a] = max(regret[I][a], 0) / rsum - else: - sigma[I][a] = 1 / len(state.legal_actions) - return sigma - - -def _create_dir(folder_id: str) -> Path: - """Create and get a unique dir path to save to using a timestamp.""" - time = str(datetime.datetime.now()) - for char in ":- .": - time = time.replace(char, "_") - path: Path = Path(f"./{folder_id}_results_{time}") - path.mkdir(parents=True, exist_ok=True) - return path - - -def cfr(agent: Agent, state: ShortDeckPokerState, i: int, t: int) -> float: - """ - CFR algo with the a temporary regret object for better strategy averaging - """ - ph = state.player_i - - player_not_in_hand = not state.players[i].is_active - if state.is_terminal or player_not_in_hand: - return state.payout[i] - - elif ph == i: - I = state.info_set - # Move regret over to temporary object and build off that - if agent.tmp_regret[I] == {}: - agent.tmp_regret[I] == agent.regret[I].copy() - sigma = calculate_strategy(agent.tmp_regret, I, state) - - vo = 0.0 - voa = {} - for a in state.legal_actions: - new_state: ShortDeckPokerState = state.apply_action(a) - voa[a] = cfr(agent, new_state, i, t) - vo += sigma[I][a] * voa[a] - - for a in state.legal_actions: - agent.tmp_regret[I][a] += voa[a] - vo - - return vo - else: - Iph = state.info_set - # Move regret over to a temporary object and build off that - if agent.tmp_regret[Iph] == {}: - agent.tmp_regret[Iph] == agent.regret[Iph].copy() - sigma = calculate_strategy(agent.regret, Iph, state) - - try: - a = np.random.choice( - list(sigma[Iph].keys()), 1, p=list(sigma[Iph].values()), - )[0] - - except KeyError: - p = 1 / len(state.legal_actions) - probabilities = np.full(len(state.legal_actions), p) - a = np.random.choice(state.legal_actions, p=probabilities) - sigma[Iph] = {action: p for action in state.legal_actions} - - new_state: ShortDeckPokerState = state.apply_action(a) - return cfr(agent, new_state, i, t) - - -def rts( - offline_strategy_path: str, - regret_path: str, - public_cards: list, - action_sequence: list, - n_iterations: int, - lcfr_threshold: int, - discount_interval: int, - n_players: int, - update_interval: int, - update_threshold: int, - dump_int: int, -): - """RTS.""" - config: Dict[str, int] = {**locals()} - save_path: Path = _create_dir('RTS') - with open(save_path / "config.yaml", "w") as steam: - yaml.dump(config, steam) - # TODO: fix the seed - # utils.random.seed(36) - agent = Agent(regret_path=regret_path) - # Load unnormalized strategy to build off - offline_strategy = joblib.load(offline_strategy_path) - state: ShortDeckPokerState = new_game( - 3, real_time_test=True, public_cards=public_cards - ) - # Load current game state - current_game_state: ShortDeckPokerState = state.load_game_state( - offline_strategy, action_sequence - ) - # We don't need the offline strategy for search.. - # del offline_strategy - for t in trange(1, n_iterations + 1, desc="train iter"): - for i in range(n_players): # fixed position i - # Deal hole cards based on bayesian updating of hole card probs - state: ShortDeckPokerState = current_game_state.deal_bayes() - cfr(agent, state, i, t) - if t < lcfr_threshold & t % discount_interval == 0: - d = (t / discount_interval) / ((t / discount_interval) + 1) - for I in agent.tmp_regret.keys(): - for a in agent.tmp_regret[I].keys(): - agent.tmp_regret[I][a] *= d - # Add the unnormalized strategy into the original - # Right now assumes dump_int is a multiple of n_iterations - if t % dump_int == 0: - # offline_strategy = joblib.load(offline_strategy_path) - # Adding the regret back to the regret dict, we'll build off for next RTS - for I in agent.tmp_regret.keys(): - if agent.tmp_regret != {}: - agent.regret[I] = agent.tmp_regret[I].copy() - for info_set, this_info_sets_regret in sorted(agent.tmp_regret.items()): - # If this_info_sets_regret == {}, we do nothing - strategy = normalize_strategy(this_info_sets_regret) - # Check if info_set exists.. - no_info_set = info_set not in offline_strategy - if no_info_set or offline_strategy[info_set] == {}: - offline_strategy[info_set] = {a: 0 for a in strategy.keys()} - for action, probability in strategy.items(): - try: - offline_strategy[info_set][action] += probability - except: - import ipdb; - ipdb.set_trace() - agent.reset_new_regret() - - return agent, offline_strategy diff --git a/research/test_methodology/average_unnormalized_strategy.py b/research/test_methodology/average_unnormalized_strategy.py deleted file mode 100644 index e35965ad..00000000 --- a/research/test_methodology/average_unnormalized_strategy.py +++ /dev/null @@ -1,91 +0,0 @@ -import collections -import glob -import os -import re -from typing import Dict, List, Union - -import click -import joblib -from tqdm import tqdm - - -def calculate_strategy(this_info_sets_regret: Dict[str, float]) -> Dict[str, float]: - """Calculate the strategy based on the current information sets regret.""" - actions = this_info_sets_regret.keys() - regret_sum = sum([max(regret, 0) for regret in this_info_sets_regret.values()]) - if regret_sum > 0: - strategy: Dict[str, float] = { - action: max(this_info_sets_regret[action], 0) / regret_sum - for action in actions - } - elif this_info_sets_regret == {}: - # Don't return strategy if no strategy was made - # during training - strategy: Dict[str, float] = {} - elif regret_sum == 0: - # Regret is negative, we learned something - default_probability = 1 / len(actions) - strategy: Dict[str, float] = {action: default_probability for action in actions} - return strategy - - -def try_to_int(text: str) -> Union[str, int]: - """Attempt to return int.""" - return int(text) if text.isdigit() else text - - -def natural_key(text): - """Sort with natural numbers.""" - return [try_to_int(c) for c in re.split(r"(\d+)", text)] - - -def average_strategy(all_file_paths: List[str]) -> Dict[str, Dict[str, float]]: - """Compute the mean strategy over all timesteps.""" - # The offline strategy for all information sets. - offline_strategy: Dict[str, Dict[str, float]] = collections.defaultdict( - lambda: collections.defaultdict(lambda: 0.0) - ) - # Sum up all strategies. - for dump_path in tqdm(all_file_paths, desc="loading dumps"): - # Load file. - try: - agent = joblib.load(dump_path) - except Exception as e: - tqdm.write(f"Failed to load file at {dump_path} because:{e}") - agent = {} - regret = agent.get("regret", {}) - # Sum probabilities from computed strategy.. - for info_set, this_info_sets_regret in sorted(regret.items()): - strategy = calculate_strategy(this_info_sets_regret) - # If strategy == {}, we do nothing - for action, probability in strategy.items(): - offline_strategy[info_set][action] += probability - # Return regular dict, not defaultdict. - return {info_set: dict(strategy) for info_set, strategy in offline_strategy.items()} - - -@click.command() -@click.option( - "--results_dir_path", default=".", help="the location of the agent file dumps." -) -@click.option( - "--write_dir_path", default=".", help="where to save the offline strategy" -) -def cli(results_dir_path: str, write_dir_path: str): - """Compute the strategy and write to file.""" - # Find all files to load. - all_file_paths = glob.glob(os.path.join(results_dir_path, "*.gz")) - if not all_file_paths: - raise ValueError(f"No agent dumps could be found at: {results_dir_path}") - # Sort the file paths in the order they were created. - all_file_paths = sorted(all_file_paths, key=natural_key) - offline_strategy = average_strategy(all_file_paths) - # Save dictionary to compressed file. - latest_file = os.path.basename(all_file_paths[-1]) - latest_iteration: int = int(re.findall(r"\d+", latest_file)[0]) - save_file: str = f"offline_strategy_{latest_iteration}.gz" - joblib.dump(offline_strategy, os.path.join(write_dir_path, save_file)) - - -if __name__ == "__main__": - cli() diff --git a/research/test_methodology/bot_test.py b/research/test_methodology/bot_test.py deleted file mode 100644 index 678470e4..00000000 --- a/research/test_methodology/bot_test.py +++ /dev/null @@ -1,152 +0,0 @@ -from typing import List, Dict, DefaultDict -from pathlib import Path -import joblib -import collections - -from tqdm import trange -import yaml -import datetime -import numpy as np -from scipy import stats - -from pluribus.games.short_deck.state import ShortDeckPokerState, new_game -from pluribus.poker.card import Card - -def _calculate_strategy( - state: ShortDeckPokerState, - I: str, - strategy: DefaultDict[str, DefaultDict[str, float]], - count=None, - total_count=None -) -> str: - sigma = collections.defaultdict(lambda: collections.defaultdict(lambda: 1 / 3)) - try: - # If strategy is empty, go to other block - sigma[I] = strategy[I].copy() - if sigma[I] == {}: - raise KeyError - norm = sum(sigma[I].values()) - for a in sigma[I].keys(): - sigma[I][a] /= norm - a = np.random.choice( - list(sigma[I].keys()), 1, p=list(sigma[I].values()), - )[0] - except KeyError: - if count is not None: - count += 1 - p = 1 / len(state.legal_actions) - probabilities = np.full(len(state.legal_actions), p) - a = np.random.choice(state.legal_actions, p=probabilities) - sigma[I] = {action: p for action in state.legal_actions} - if total_count is not None: - total_count += 1 - return a, count, total_count - - -def _create_dir(folder_id: str) -> Path: - """Create and get a unique dir path to save to using a timestamp.""" - time = str(datetime.datetime.now()) - for char in ":- .": - time = time.replace(char, "_") - path: Path = Path(f"./{folder_id}_results_{time}") - path.mkdir(parents=True, exist_ok=True) - return path - - -def agent_test( - hero_strategy_path: str, - opponent_strategy_path: str, - real_time_est: bool = False, - action_sequence: List[str] = None, - public_cards: List[Card] = [], - n_outter_iters: int = 30, - n_inner_iters: int = 100, - n_players: int = 3, - hero_count=None, - hero_total_count=None, -): - config: Dict[str, int] = {**locals()} - save_path: Path = _create_dir('bt') - with open(save_path / "config.yaml", "w") as steam: - yaml.dump(config, steam) - - # Load unnormalized strategy for hero - hero_strategy = joblib.load(hero_strategy_path) - # Load unnormalized strategy for opponents - opponent_strategy = joblib.load(opponent_strategy_path) - - # Loading game state we used RTS on - if real_time_est: - state: ShortDeckPokerState = new_game( - n_players, real_time_test=real_time_est, public_cards=public_cards - ) - current_game_state: ShortDeckPokerState = state.load_game_state( - opponent_strategy, action_sequence - ) - - # TODO: Right now, this can only be used for loading states if the two strategies - # are averaged. Even averaging strategies is risky. Loading a game state - # should be used with caution. It will work only if the probability of reach - # is identical across strategies. Use the average strategy. - - info_set_lut = {} - EVs = np.array([]) - for _ in trange(1, n_outter_iters): - EV = np.array([]) # Expected value for player 0 (hero) - for t in trange(1, n_inner_iters + 1, desc="train iter"): - for p_i in range(n_players): - if real_time_est: - # Deal hole cards based on bayesian updating of hole card probs - state: ShortDeckPokerState = current_game_state.deal_bayes() - else: - state: ShortDeckPokerState = new_game(n_players, info_set_lut) - info_set_lut = state.info_set_lut - while True: - player_not_in_hand = not state.players[p_i].is_active - if state.is_terminal or player_not_in_hand: - EV = np.append(EV, state.payout[p_i]) - break - if state.player_i == p_i: - random_action, hero_count, hero_total_count = _calculate_strategy( - state, - state.info_set, - hero_strategy, - count=hero_count, - total_count=hero_total_count - ) - else: - random_action, oc, otc = _calculate_strategy( - state, - state.info_set, - opponent_strategy, - ) - state = state.apply_action(random_action) - EVs = np.append(EVs, EV.mean()) - t_stat = (EVs.mean() - 0) / (EVs.std() / np.sqrt(n_outter_iters)) - p_val = stats.t.sf(np.abs(t_stat), n_outter_iters - 1) - results_dict = { - 'Expected Value': float(EVs.mean()), - 'T Statistic': float(t_stat), - 'P Value': float(p_val), - 'Standard Deviation': float(EVs.std()), - 'N': int(len(EVs)), - 'Random Moves Hero': hero_count, - 'Total Moves Hero': hero_total_count - } - with open(save_path / 'results.yaml', "w") as stream: - yaml.safe_dump(results_dict, stream=stream, default_flow_style=False) - - -if __name__ == "__main__": - strat_path = "test_strategy2/unnormalized_output/" - agent_test( - hero_strategy_path=strat_path + "random_strategy.gz", - opponent_strategy_path=strat_path + "offline_strategy_1500.gz", - real_time_est=False, - public_cards=[], - action_sequence=None, - n_inner_iters=25, - n_outter_iters=75, - hero_count=0, - hero_total_count=0 - ) diff --git a/research/test_methodology/entry_rts_ab.py b/research/test_methodology/entry_rts_ab.py deleted file mode 100644 index df29136e..00000000 --- a/research/test_methodology/entry_rts_ab.py +++ /dev/null @@ -1,67 +0,0 @@ -import numpy as np -import json -import joblib - -from RT_cfr import rts -from bot_test import agent_test -from pluribus.poker.deck import Deck - - -check = joblib.load('test_strategy2/unnormalized_output/offline_strategy_1500.gz') -histories = np.random.choice(list(check.keys()), 2) -action_sequences = [] -public_cards_lst = [] -community_card_dict = { - "pre_flop": 0, - "flop": 3, - "turn": 4, - "river": 5, -} -ranks = list(range(10, 14 + 1)) -deck = Deck(include_ranks=ranks) -for history in histories: - history_dict = json.loads(history) - history_lst = history_dict['history'] - action_sequence = [] - betting_rounds = [] - for x in history_lst: - action_sequence += list(x.values())[0] - betting_rounds += list(x.keys()) - action_sequences.append(action_sequence) - final_betting_round = list(betting_rounds)[-1] - n_cards = community_card_dict[final_betting_round] - cards_in_deck = deck._cards_in_deck - public_cards = np.random.choice(cards_in_deck, n_cards) - public_cards_lst.append(list(public_cards)) - -for i in range(0, len(action_sequences)): -# try: - public_cards = public_cards_lst[i].copy() - action_sequence = action_sequences[i].copy() - agent_output, offline_strategy = rts( - 'test_strategy2/unnormalized_output/offline_strategy_1500.gz', - 'test_strategy2/strategy_1500.gz', public_cards, action_sequence, - 1500, 400, 400, 3, 400, 400, 20 - ) - save_path = "test_strategy2/unnormalized_output/" - last_regret = {info_set: dict(strategy) for info_set, strategy in agent_output.regret.items()} - joblib.dump(offline_strategy, save_path + f'rts_output{i+35}.gz', compress="gzip") - joblib.dump(last_regret, save_path + f'last_regret{i+35}.gz', compress="gzip") - - public_cards = public_cards_lst[i].copy() - action_sequence = action_sequences[i].copy() - strat_path = "test_strategy2/unnormalized_output/" - agent_test( - hero_strategy_path=strat_path + f"rts_output{i+35}.gz", - opponent_strategy_path=strat_path + "offline_strategy_1500.gz", - real_time_est=True, - public_cards=public_cards, - action_sequence=action_sequence, - n_inner_iters=25, - n_outter_iters=250, - hero_count=0, - hero_total_count=0, - ) -# except: -# print(f"ERROR on {action_sequences[i]}, {public_cards_lst[i]}") -# pass diff --git a/research/test_methodology/validating_nash_equilibriums_via_simulations.ipynb b/research/test_methodology/validating_nash_equilibriums_via_simulations.ipynb deleted file mode 100644 index c146c559..00000000 --- a/research/test_methodology/validating_nash_equilibriums_via_simulations.ipynb +++ /dev/null @@ -1,224 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Validating Nash Equilibriums Via Simulations" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "_by Colin Manko_" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In an effort to validate and test possible improvements to core poker artificial intelligence algorithms, I have designed the following methodology." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Goals" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "- Validate that MCCFR offline learning strategy is approximating a Nash equilibrium\n", - "- More generally, create a methodology that allows for rigorously testing changes made to the core AI algorithms" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Prerequisites" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "- Need two test bot implementation strategies ($\\beta{1}$ and $\\beta{2}$) that we would like to compare\n", - "- Need a human tester ($H_{0}$) as a quasi control. The human tester should not have access to any underlying strategies from the test bots or simulated Nash" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Step 1: Randomly Generate Test Game" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Given a set of $N$ game tree nodes (this is the entire game tree, as given by infoset, $I$), randomly generate $x$ test nodes without preplacement and with equal probability. Call the set of test nodes $U$.\n", - "\n", - "As a side note, we will account for probability of reach ($p(h)$) in another step. Equal probability across nodes allows us to find patterns across nodes where our agent underperforms. We will adjust the expected value at $I$, ($v^{\\sigma}(I)$), by $p(h)$.\n", - "- **How to**: For Limit Texas Hold'em, the number of action sequences ($N$), is small enough that they can be found computationally rather than analytically. We can run *all_action_sequences.py* in the *size_of_problem* directory to generate this list. \n", - "- _Something like 15-20 hours and less than 4GB??_\n", - "- Generate $x$ integers to be indices and select them from the *all_action_sequences.py* output\n", - "- Once $x$ action sequences are generated, randomly generate $x$ public card combos, based on the betting stage of the test node, $u$, as well as one pair of private hole cards to be used by $\\beta{1}$, $\\beta{2}$ and $H_0$. They will only get that hand at $u$. " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Step 2: Prepare Realtime Search for Finding Nash Equilibrium" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For each test node, $u$, in $U$, use realtime search to compute the Nash Equilibrium ($\\sigma^*$) by constraining the search algorithm to start at $u$, where $u$ is equivalent to $I$ in regard to action sequence, but does not have any set hand for the traversing player ($p_i$).\n", - "\n", - "Use a pooled strategy between $\\beta{1}$ and $\\beta{2}$ to estimate $p(h)$ without bias:" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For hand in possible combinations of real hands:\n", - "
\n", - "    For idx, $a$ in action sequence at $u$:\n", - "
\n", - "        if idx == 0:\n", - "
\n", - "             $p(h)_\\beta{1}$ = $\\beta{1}$[$I$][$a$]\n", - "
\n", - "             $p(h)_\\beta{2}$ = $\\beta{2}$[$I$][$a$]\n", - "
\n", - "        $p(h)_\\beta{1}$ *= $\\beta{1}$[$I$][$a$]\n", - "
\n", - "        $p(h)_\\beta{2}$ *= $\\beta{1}$[$I$][$a$]\n", - "
\n", - "    p(h)[rs] = ($p(h)_\\beta{1}$ + $p(h)_\\beta{2}$)/2" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The root node of the realtime search algorithm is replaced with a chance node that represents each possible node in the public state $G$ [[Brown, Sandholm, Amos]](https://papers.nips.cc/paper/7993-depth-limited-solving-for-imperfect-information-games.pdf). From the above psuedo-code, this deal can be generated as: " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Generate deal order\n", - "
\n", - "For $i$ in $P_i$ deal order:\n", - "
\n", - "    Generate hand for player based on normalized $p(h)[rs]$ if available, else try again" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "_The \"if available, else try again\" part could be made better_" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Other important features of the _\"Nash Bot\"_ real time search..**:\n", - "- The _\"Nash bot\"_ is the master of this node. In order to reach full convergence, from the normal MCCFR algorithm, we must remove the sampling of actions for opponents.\n", - "- For ease, the real time search should not use leaf nodes, but should search to the end of the game tree, where either a terminal node or a shown down is entered. In this way, we can get a truer sense of the expected value." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "A Nash Equilibrium is found if the change in strategy on each iteration drops below some threshold $t$ for the real hand we are testing for. Charting probabilities for each action in $u$ over time for the randomly generated real hand to test should show a convergence over time." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "_One main benefit of using this real time search to validate CFR is this search will need to be developed anyway._" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Step 3: Test and Measure Success" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**For the test bots:**\n", - "For each $u$ in $U$, play each test strategy ($\\beta{1}$ and $\\beta{2}$) against the _\"Nash bot\"_ for $r$ number of simulations. The _\"Nash bot\"_ should be dealt available hands from the distribution of probabilities as determined by $p(h)[rs]$ in the pseudo-code above. Both the test bots and the human tester will be dealt the same hand in each simulation of game play on $u$, as randomly generated in step 1. \n", - "\n", - "If $\\beta{1}$ or $\\beta{2}$ has converged to a Nash equilibrium, then we should expect $v^\\sigma$ to be equal to 0 for our test bot, assuming that _\"Nash bot\"_ has converged to a Nash equilibrium itself. $v^{\\sigma^*}(u)$ and $v^{\\sigma}(u)$ are the estimated payouts for the _\"Nash bot\"_ opponents and the \"hero\" (test bots or human), respectively.\n", - "\n", - "**For the human tester:**\n", - "We can simply create a contrived game. Based on the normalized probability of reach for $u$, $\\bar{p(h)}$, we can randomly generate which $u$ the human player is entered into, however they will always have the same hand upon entering $u$ and their opponents hands will vary based on $p(h)[rs]$.\n", - "\n", - "The test metric is as follows, after $p(h)$ has been normalized for space $U$, $\\bar{p(h)}$:\n", - "$$\\sum_{i=1}^{x}(v^{\\sigma}(u_i)-v^{\\sigma^*}(u_{-i}))\\times{\\bar{p(h)}}$$" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The value closest to 0 (summing no testing agent goes over 0) will have best approximated the Nash equilibrium. Additionaly, $H_0$ can be used as a quasi-control, to validate that the bot is beating a human.\n", - "\n", - "The above metric also has some degree of simulation error. For each simulation in $r$ simulations, we create a distribution of values that has a standard deviation and follows the normal distribution. \n", - "\n", - "Along with calculating the expected payout per simulation, $u^{\\sigma}(u_i)-u^{\\sigma^*}(u_{-i})$, we can also calculate $\\sigma$ for this distribution in order to describe a confidence interval around the test metric. \n", - "\n", - "Finally, a simple difference of means can be done between each test bot to decipher a winner and if that winner had a statistically significant edge. We can then study each $u$ in $U$ to find patterns in which nodes the espspective bots did not do well with." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.0" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/research/to_do.md b/research/to_do.md deleted file mode 100644 index e510af90..00000000 --- a/research/to_do.md +++ /dev/null @@ -1,49 +0,0 @@ -A Place for Next Steps in Short Deck Implementation - -## Abstraction - -#### Information Abstraction -- hard code opening hand clusters -- decide how to store these for lookup in blueprint/real time algo -- run for short deck - -#### Action Abstraction -- not sure how this fits into blueprint/real time yet - -## Blueprint Algo -- apply to contrived short deck game - -## Real Time Search Algo -- need isomorphic/lossless handling of cards?? # Non-essential maybe.. -- mock up "toy" version - - pre-req: stateful version of short deck - -### Rules of Contrived Short Deck Game -- 3 players -- 2-9 removed -- no adjustments to hand rankings versus no-limit -- 10000 in stack, 50 small blind, 100 big blind -- limited betting - -#### Possible Next Steps -- fix short deck game and roll out to online hosting? -- go right on to full game? - -#### Current (Concise) Papers -- Abstraction - - https://www.cs.cmu.edu/~sandholm/hierarchical.aamas15.pdf <- this algo - - http://www.ifaamas.org/Proceedings/aamas2013/docs/p271.pdf <- these features -- Blueprint - - https://science.sciencemag.org/content/sci/suppl/2019/07/10/science.aay2400.DC1/aay2400-Brown-SM.pdf <- pseudo code -- Real Time Algo - - https://papers.nips.cc/paper/7993-depth-limited-solving-for-imperfect-information-games.pdf <- build off this - - make theses changes: - - [optimized vector-based linear cfr?](https://arxiv.org/pdf/1809.04040.pdf) - - [only samples chance events?](http://martin.zinkevich.org/publications/ijcai2011_rgbr.pdf) - -#### TODO: Colin -- Generate abstraction for 20 cards --- Program to turn that into dictionary and store separately -- Hard code preflop lossless -- Write next steps in docstring of blueprint algo -- Consider getting rid of notebooks before merging into develop.. \ No newline at end of file From 8aa9c09d56c96d9d01e81a90be86ed0017437096 Mon Sep 17 00:00:00 2001 From: big-c-note Date: Sat, 13 Jun 2020 11:34:29 -0400 Subject: [PATCH 41/42] shortening tests and test data, will make private data for fuller regressions --- ...ng_nash_equilibriums_via_simulations.ipynb | 224 ------------------ .../data}/action_sequences.pkl | Bin .../data}/random_action_sequences.pkl | Bin test/data/random_offline_strategy.gz | Bin 0 -> 40297 bytes test/functional/test_short_deck.py | 73 +++--- 5 files changed, 40 insertions(+), 257 deletions(-) delete mode 100644 research/stat_tests/validating_nash_equilibriums_via_simulations.ipynb rename {research/size_of_problem => test/data}/action_sequences.pkl (100%) rename {research/size_of_problem => test/data}/random_action_sequences.pkl (100%) create mode 100644 test/data/random_offline_strategy.gz diff --git a/research/stat_tests/validating_nash_equilibriums_via_simulations.ipynb b/research/stat_tests/validating_nash_equilibriums_via_simulations.ipynb deleted file mode 100644 index c146c559..00000000 --- a/research/stat_tests/validating_nash_equilibriums_via_simulations.ipynb +++ /dev/null @@ -1,224 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Validating Nash Equilibriums Via Simulations" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "_by Colin Manko_" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In an effort to validate and test possible improvements to core poker artificial intelligence algorithms, I have designed the following methodology." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Goals" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "- Validate that MCCFR offline learning strategy is approximating a Nash equilibrium\n", - "- More generally, create a methodology that allows for rigorously testing changes made to the core AI algorithms" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Prerequisites" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "- Need two test bot implementation strategies ($\\beta{1}$ and $\\beta{2}$) that we would like to compare\n", - "- Need a human tester ($H_{0}$) as a quasi control. The human tester should not have access to any underlying strategies from the test bots or simulated Nash" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Step 1: Randomly Generate Test Game" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Given a set of $N$ game tree nodes (this is the entire game tree, as given by infoset, $I$), randomly generate $x$ test nodes without preplacement and with equal probability. Call the set of test nodes $U$.\n", - "\n", - "As a side note, we will account for probability of reach ($p(h)$) in another step. Equal probability across nodes allows us to find patterns across nodes where our agent underperforms. We will adjust the expected value at $I$, ($v^{\\sigma}(I)$), by $p(h)$.\n", - "- **How to**: For Limit Texas Hold'em, the number of action sequences ($N$), is small enough that they can be found computationally rather than analytically. We can run *all_action_sequences.py* in the *size_of_problem* directory to generate this list. \n", - "- _Something like 15-20 hours and less than 4GB??_\n", - "- Generate $x$ integers to be indices and select them from the *all_action_sequences.py* output\n", - "- Once $x$ action sequences are generated, randomly generate $x$ public card combos, based on the betting stage of the test node, $u$, as well as one pair of private hole cards to be used by $\\beta{1}$, $\\beta{2}$ and $H_0$. They will only get that hand at $u$. " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Step 2: Prepare Realtime Search for Finding Nash Equilibrium" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For each test node, $u$, in $U$, use realtime search to compute the Nash Equilibrium ($\\sigma^*$) by constraining the search algorithm to start at $u$, where $u$ is equivalent to $I$ in regard to action sequence, but does not have any set hand for the traversing player ($p_i$).\n", - "\n", - "Use a pooled strategy between $\\beta{1}$ and $\\beta{2}$ to estimate $p(h)$ without bias:" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For hand in possible combinations of real hands:\n", - "
\n", - "    For idx, $a$ in action sequence at $u$:\n", - "
\n", - "        if idx == 0:\n", - "
\n", - "             $p(h)_\\beta{1}$ = $\\beta{1}$[$I$][$a$]\n", - "
\n", - "             $p(h)_\\beta{2}$ = $\\beta{2}$[$I$][$a$]\n", - "
\n", - "        $p(h)_\\beta{1}$ *= $\\beta{1}$[$I$][$a$]\n", - "
\n", - "        $p(h)_\\beta{2}$ *= $\\beta{1}$[$I$][$a$]\n", - "
\n", - "    p(h)[rs] = ($p(h)_\\beta{1}$ + $p(h)_\\beta{2}$)/2" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The root node of the realtime search algorithm is replaced with a chance node that represents each possible node in the public state $G$ [[Brown, Sandholm, Amos]](https://papers.nips.cc/paper/7993-depth-limited-solving-for-imperfect-information-games.pdf). From the above psuedo-code, this deal can be generated as: " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Generate deal order\n", - "
\n", - "For $i$ in $P_i$ deal order:\n", - "
\n", - "    Generate hand for player based on normalized $p(h)[rs]$ if available, else try again" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "_The \"if available, else try again\" part could be made better_" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Other important features of the _\"Nash Bot\"_ real time search..**:\n", - "- The _\"Nash bot\"_ is the master of this node. In order to reach full convergence, from the normal MCCFR algorithm, we must remove the sampling of actions for opponents.\n", - "- For ease, the real time search should not use leaf nodes, but should search to the end of the game tree, where either a terminal node or a shown down is entered. In this way, we can get a truer sense of the expected value." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "A Nash Equilibrium is found if the change in strategy on each iteration drops below some threshold $t$ for the real hand we are testing for. Charting probabilities for each action in $u$ over time for the randomly generated real hand to test should show a convergence over time." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "_One main benefit of using this real time search to validate CFR is this search will need to be developed anyway._" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Step 3: Test and Measure Success" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**For the test bots:**\n", - "For each $u$ in $U$, play each test strategy ($\\beta{1}$ and $\\beta{2}$) against the _\"Nash bot\"_ for $r$ number of simulations. The _\"Nash bot\"_ should be dealt available hands from the distribution of probabilities as determined by $p(h)[rs]$ in the pseudo-code above. Both the test bots and the human tester will be dealt the same hand in each simulation of game play on $u$, as randomly generated in step 1. \n", - "\n", - "If $\\beta{1}$ or $\\beta{2}$ has converged to a Nash equilibrium, then we should expect $v^\\sigma$ to be equal to 0 for our test bot, assuming that _\"Nash bot\"_ has converged to a Nash equilibrium itself. $v^{\\sigma^*}(u)$ and $v^{\\sigma}(u)$ are the estimated payouts for the _\"Nash bot\"_ opponents and the \"hero\" (test bots or human), respectively.\n", - "\n", - "**For the human tester:**\n", - "We can simply create a contrived game. Based on the normalized probability of reach for $u$, $\\bar{p(h)}$, we can randomly generate which $u$ the human player is entered into, however they will always have the same hand upon entering $u$ and their opponents hands will vary based on $p(h)[rs]$.\n", - "\n", - "The test metric is as follows, after $p(h)$ has been normalized for space $U$, $\\bar{p(h)}$:\n", - "$$\\sum_{i=1}^{x}(v^{\\sigma}(u_i)-v^{\\sigma^*}(u_{-i}))\\times{\\bar{p(h)}}$$" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The value closest to 0 (summing no testing agent goes over 0) will have best approximated the Nash equilibrium. Additionaly, $H_0$ can be used as a quasi-control, to validate that the bot is beating a human.\n", - "\n", - "The above metric also has some degree of simulation error. For each simulation in $r$ simulations, we create a distribution of values that has a standard deviation and follows the normal distribution. \n", - "\n", - "Along with calculating the expected payout per simulation, $u^{\\sigma}(u_i)-u^{\\sigma^*}(u_{-i})$, we can also calculate $\\sigma$ for this distribution in order to describe a confidence interval around the test metric. \n", - "\n", - "Finally, a simple difference of means can be done between each test bot to decipher a winner and if that winner had a statistically significant edge. We can then study each $u$ in $U$ to find patterns in which nodes the espspective bots did not do well with." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.0" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/research/size_of_problem/action_sequences.pkl b/test/data/action_sequences.pkl similarity index 100% rename from research/size_of_problem/action_sequences.pkl rename to test/data/action_sequences.pkl diff --git a/research/size_of_problem/random_action_sequences.pkl b/test/data/random_action_sequences.pkl similarity index 100% rename from research/size_of_problem/random_action_sequences.pkl rename to test/data/random_action_sequences.pkl diff --git a/test/data/random_offline_strategy.gz b/test/data/random_offline_strategy.gz new file mode 100644 index 0000000000000000000000000000000000000000..ff4cfe7fcb4234989674ade4e7e8a3ba3c35e2c0 GIT binary patch literal 40297 zcmV*bKvcgUiwFP!000006YQM{oR!lX$0v%U5~ZThMiZ5pJ9~?{P3pEuMv=XyV!D;O zuS#g_#!kqdvacb85VB+qA%u`UgploT`@iRR&pr3Nz3WPTjI9k(E8}*cnr%ozQdKRBJ-dDKjQdpE%aC#-7mRxN&1AkF}a48YHhb z89)7mX%m_xwiw+oD=TYMZeB@Z%b{6A8=ReJSe|H_4ea;(B(@qpeE5Fdw;0-R zXyfULtt-m9Z;_RCl6|F<{X5S79c%wu_V0N6_c;6aDEj_n`~PRq_omzbKY{*t&H^Jx2*d?vY%TuXfpxreVS z`ESTCc`x5RB8@wC;&Bra&0|rb68mRHWrNC; zSPP00+a<-?qCBy^E!IlNbUKyk4i#n74aqd0lS!r}Dw2}!^&q4us$*?+ATlTT5)6qs zA((mLK+Fj-E#$L~RYYA>KVrfrQQT8zq&73*l6%YQM62?|j#Y`)oZQ=Z$h~c>^zRw@ zQ@4pOADx!_PO;J@3m@5T(oc7Ba*yQXCU#EByTklfrC!{P1H|>N)XS8dhWvz z`TB`S)3F48>&IfS?CM0f^2DB1iM=@0clS_zk63i*smZu3?UZ`5|B#QtO7atXCsD{L zPvkNb8dIkfsVHk~&?)6?CQiwOQ5SUwL~=J#2a()Gbr4-QQO8g@p1kTretDvxDp8nS z*_}~V*~5!167Urz*(xqil(+)EUgiS6T$`&P`#3sg(1a#*+0oIh&b_j!f=qK6#COo) z))nyeu1@qRPxP%y?8CXfQV-V`jYT``_7C~tTwlLfwB0h|4!7X@W4Q&1vLv4U%M<%D zJP)U?Z$L%a;Ran_kaz~symg77DU32@#&n*)&b_kFMF<Y^w|Ffz^pvd16piVlapG zkXW=Ijb2$+ai~{TahPXBn_ryRFNyW=^2GiO>k+hbH=?3!grUw|kkPEF*EHQR6UkC^ zUA6wEGlr&7cS-qx>O^^Y;=roJNKVS5JUrAvvC`ij?Rox<>z?5})WNZ6&voS8&m9@V zoW$s)ln*IS9Ll8JfqJOJD#|(-^icn04AWy_{1}p-v79jAAgY7tx@lxoO_Ge4s?FoE zyvgC!i6hDrM^+_{;?Nx9f#%p)>4s#skVCV=)5{b^662C+jxSG4U}!F)UgqeEvPFix z4C&j|hapzk;6oHR>c}pWk~We4%yhp_JH>cRb)vF7F|jHU=TMpCfy(4qH22r?X)6cB zIaH>^qJ@p`zIg9PFXK~;#f6Dulc-EBPfTN|tfQzTD$3RwLM39Ok{6;BqmCR+;Uubq z?)AD3>_qi9Q9l_^R8u`%fudENIIcV~y(%$-Lu{r8V#nL5j3e3%czya29AdL#rC*ca z@zER}vAjs)gd}1omM2bPh_#@I&8{eGVF)pj#t0HI)tLsWL59d1sajOk&lh=wOng2I zjH;>=CzmHqsY;y6N%1rfDV}ai@yF_=H{6@eN%4$WwCtH9N4zsP&oQ8slQ=Ue#k0y2 z)l7=NQ7N8XQTCf5DU!e_NGa-An!36-C@*5d%8F1>{R~wUWvslaIdWP5bE*^PmM6}u zN}QivnKt^B$1~_EXT_pq)ZPB!vC7X&T#&@-!t%sL468mAtBWhj`WV8B1W{p1R*TRP zU00QjpYFcuXQ)b3gofxD)HWxNd4oCCiA%~8msTY%+S}!cB00s08jnH<3?=8 z!b9oz(sOVh_MU@VoLG{?@{#hyqYTTo6wAjd%Gw&jGECl72P+*(2ZfypZ?EHNb)J#v zUK4?~46ahcyGyGRkC!K&s7gG^A-T-MQ$7`ocAnX_VrJKIoTq#`7R^3)^>yd2Ua-lS z)iX&XpDj-;ca2#+SAWN>RAp%@I8*&xKmA#eY;{bIiTdl{To-j`Uhnzp#0%w#7poF4 zafrR_f!GSKe&j2$XbX2nvmih5Y7(*6$`dOYVwE(b`FcfJrJ;;w$$x<;TjVdhaNZAo znUbgoYteNnFY-op;?45Js;b0WWMxNEu31_3%8t}hDu;V0PQ0De?mOj))l9n^sCM72 zDBEC2yCf4Irfhs|o?p}sqU(NxbCm^#s-Ge2{#48LGrv`*6Q%R)rE?bVRVUsrPpqj* ze89Ph4{d@r?HqmN*$=YUORY^(^>KOP6GqkC2FV5TpH`I3wRbZyv296MW^RjGMReUn zP$HTEA!R*3t4@4gp7^3F@g>*t>pTL4uVT^0t|^vZdoQ2MOMH`*!}{{Xw@eP*X}9=0 zdlwsn-Qpm#RjSPkzNm_JW$7HB}cRvovuUSfcmes(PSGBc;{cpLorTuS})zJPoyD~jx zsM4M(G=O%bukc>1S!6Y`Z-R)etx(j~q>$8JaKosS8$-&&3`#jHBa>m5kuD~g!obgS z+jKUjwGAZP1iIT65^hS=$96chn%OViy%?)Gc{^Hms%x5NQIWMB({l@yyFIVxXQ`fd zfS#WvVReKH=`J^khzumD>bB?tqlk{FFsyi>~(NxksAnDNtC0!KY(5;Rn{WO(o zbRddOf~bQ?vb8-!KM|Zu&OIULy`jMz$T`<(#v|TlJg){bo)0rlnqrd$R|Pp%0n>aT zKCj42^TkFrPr{KPMFxI`)4T3h zdhfMwN7sFsocBRdrM#R|(RCDZHWgjhr+q2?=EGEGAJitfOr)1}RwT%{CzcEB2Z@$J zSN$Q;eN`nIu_ZcyNpv8Jit!R%njB`b20@}r4UIGrYg-?-*AZD_-RmZ-p>Bkd$?srl zC4;Drb%n8Pr$c6xEEC_+70C^T$ru98hr(nGb4GIec}H@?Yl!6bMG0- z^nMWdIv6h5F}an^bM5Ih&oz>hW03>IlEZU{pr}K6bm&f$)?q-$)Mk`nLr15RBQm8y zxLwuHP?fw0Ingr^U01EY>1Fk)=i$eMy4>F3!14&NbR@7m%IWsTc)Pu^aBs!Qd~qRs;MrYIlXgdg-Svi&b|jn*GJ&D04O7FBIyOPI&VI&Qf?!I- zz0NZd)!%P#o%L9c0f&{~Vj?(&Ee!4!p|w-u&+lVvDiA6aX1y9H;w0T7UeJj z4rdwSkgORAay&>?)2cp3)V1{c`68W(uB(2opP5>sI{3Zr@=X?)I1U<}4kl(GWPC8a z9?D+UPNd~bmfL&6STmW%kH`J9c#YG|VXPCNaZ~%l1U=@NNqeSKwh0YpiZ4^$HoqFz ziQQCfluXyp@Aq)AdLmdo39QWqt5sk%eJD<)eHPBHboQlv0uKGtv8iCObuz>D6clwT zk1ahj!#WMvnmRV4zPKj8O(h#*27zr@zXJ=yPKS)ofcDOWjL&k0Vb$JY*x3lT$Z)B% zJ9G~JpF6rkIo7#M>*wL~&gZqhl7?XyK6$j>fy2^gM+(VI=IUvj(x_8EwWNK znKrx}pLYe%8r|8*x)Q9J+T1AU)PqQ`IxOmxEgN!)LOxZVRKumPvL^fdWWqhw(FRwT zcNLhu8Vp?nX0LTJJI|Zh>l|;Vr>)Khvjg1IRts~i1&rD2@p(7!%$`d9#RE>8$Tmm;4ds)O!zzna_JV?_S@$*`M=IM~JaJ>dIZ z@U{qi-$$t1WP7Hfdyc>0uaXxk?GfzK1I{^ai|2|R-M#|reunM?C~7f}?q}4kJqUC^ zGvwAtC>rFXG7%Q4)3P%$;-n)P*^o@f?FCUk8MH)g>8GA9v>pOl4}*s#KwFZx za0tz>q;|En9;?B1F6H0n`O7V~9%uYLfzNxA=Wlg#b0=#V_*-rNYhr&VGBQAZ!?MU* zHQdSse?fOgU7C9entK`=cm|q#)~UJW-r?GFkRn;*yy+5+66<-Utrt-4i@dgGQ``O$ zv^CqHZ4Z;SMD{{-Jq+3mqK-Pg!B72M`hOYvUjd!I0{y@05zY)vcFpZ@CMkI5F;Mp6 z($|>eSE8ubdC8|1m%ahXn_65-GT31XXG}!#qMnE!H!C~x<2Nf**S!#zq~C<3S3!qw zLDFwKP5L|DCY|{CShRJ!CnisC)Qrn0zl--{OV5cUPC3#M#z(C8nC{=l=dE#s5!BRw zkPIUXn);wZ2vswznvoId-H(Y=twSbFa1iyE3Ey?XtlGLcY{c|KF!vD{SPSMpb~5*g zH*=qYt%0r))6e+-+%bxiZ+*^~`vRZ$CC^-An)O--<{BHydW9*pQf1Ol*sj_&m-R6s z9(4R$*t_F(_1zhxUxCrD!OS;cbUi}7^wDdT*Rzq+O8cC(UCG$$l*P_H#;tD|e&3;} z?|J;_X5-cd;Ad*TaguNja^9$@sYKW78d1~c-W1*@o{Pv`>$HvN-rAOHGR%GeW3OEsA7Ihck*4*<6^hzY zblrqjMd7XLXP6Fc{j435wFN=Aa+%OqryezF#MR~e6U_ew2LA^0{}2-6j(uO^*zP5| z(~$DHw{#m6Pj2$E>w|YsI`r)&n@oVsvipk3iAoLZzsjvG?0?zzqp1n7TiS1$n*bZO ziAp+riterJRE>^*4n)rlZE00oji9})?0>7Qt%(BTj$T~c(RoWM*zRv-jk4FaH)aCf z28A@?1)NWZSGI+K2?wTzSHct|>KIq4K6g_zQZqc?=4hnt$WyKyl3o}yG#2egQ@6uB zH_y*2uv)OD+8#yiz&F)G+MRBRrdn7JyVE+Xh4Eh5ZmuZy5ZOr4b)9Ep!v2c-0kzdn z$A-#2f0@|CbRAuBNh_F$9l>>Ln20un?(|mDZS9vHTS@Ok-i~&4rR8?!-*=3t=UDBS z9chox+l99yOQ^};6?TLS?wc_AWC$oMn=z`!@_y2CA|`dOOZ9JegQhy5jXOe9o#1vv z=hH=UBUWdokuJD@cU~hSsIS=r8W~~G*96&T%~a}C*6xVxzUnYvnEHDnn_r*5n~c9u z{eCWg&=u@vgS~EGcTYFFsUdzcd65qM^QOMAz}kypO!j0fw|ejx&!wZDJ%RCD`(G2L zzP`*k7MbA9!A+=|xJ+y>ppC-bKp_YE%>@b(XK0k?9UA2$1fqNMaq`1g5U@*-9t z=7R}{?#GRtJlQJ6+u64^voDO~v>N55&1p4~TVnNNS}nup_2;!ZnTBZl zLaUPvhG=1!(Nb4B4I;GD(V?jsUOFiQzxLkb8j?)aUwW8f0PMg(a2bOg7)0odJ0`Tn zlk?aLCl7yi{qPs!$=PfBTy|gYmiIXpq2%RTgBj~XQ07pc^|sV6k@L;$_0R_WQkc%r z)L}jdZcW6Crn^n9ZpD7k<#1?df9R5I%Thy1poU4r`Nh@&OpfI!>OfwOsZlR-a-`W& zum4<6R@P)YWFWtt>|Z~bsUX6zjLUcwWPA{`cQ9m3PL0W~T$UbCc`E(q)3%NEmtUK^ktqp=0bQN%CaS;ax+*%ENb>4@e{Dpk0nr4wvViDuPFFeI+f~kh zOKh`r?^#D~c!G14GvPvi*sg5XGy67nB;5+F;~B?ftL1X*1fJtDl;ab@@fbrKlLT>n zSw87UtD9plM)Vv+*L6ZuS(mIkjw$*MqU(OXqbs3t64)n;^UJL&uz#|X{ZqWzCp(bY zZrj~0EY7h`V~n4Ua?juyPi^#hCKxxl(Pxm!HdUJ|8*b^exvXwQVYF%jTK2p7(W~io z06KM$PF|k{rmCU+v%%CkPNvTFX6igJH8?qqmGh_P^Zz;D^(@M_E?}%(h|jx-XKe{h z!CVa1me~KA8gM1sfdrLJ6fvO_JL!~diZ|8xQNY)!^+KpnI(wT{acZbFd*&5V7Vlbgx5E^LcbP z(4ON0pu53P&yh^U3scXrt}&@A41zEpItf-${dBLJXm^?3s_LMUY$W2xjEKs1nVIWh zW^RCOxDjUNCTeC$l!7KEUVOwg@^mZ_G2N<{$4BbUYh-21e%057i@?);(Ek14=>hnpPU*qV&ar5h73Aj?F5|NpOf|&&ukuVq@~sCMQxBoshaGV+ zU42M$J$4*yaP?u3Ol8Vn$?8Uj7m=n!*HvRMm*yUU<{pJM9)sqVLUV1>nrj=2Qe$4Z zljmajNS^gLli3p}>PcQ^vuSs08DuuwP@=Gf6Yp4`a0+mGGOn6aQE=WLe4+3Z1< z&W!np-`u3YdX5=^=TZ0zyb+!T3@iDj_Z?GFdx)NcT zpCHn{=(?(IGvVfR>_QOVPsF}X9YptvuKSG?RTuP1cNabZnq0p*Z90|c>X8TkrlEi#_STY zu!N{0ZRlPP!ZxUWhOFPrq)n09-|Jg&@EvsdJvboSxMx@Xl+KiGv^SM-&vCU6>-~Z0 z`bXUV6R+#kVZ9rnYg32y))!5y>LqA@B^yg;(x=GQK_-nPrS+eo^S2vyLA|Dnu3c>S-SaowNL{|ZBKU6}f6I-4noY`*IJ zYd`#CVn#;F?whE;h&|QMko8GA&&4FuVfucq5vIRjOa2D;|G<`H*@$nNxhy{AAM!&| zKKr2C*ub&%dx_P+z9~5ucMJQka%)Rc$j~h-w;Dct_x}EK9xu9e<64|?-MZ|BL$>YU*!C{Z z+J;HF35wd5mvSpA<))A_*{Ilrlu0xgromE?R})!Z>L}H**b%aA4Xw3-Y}-P%lVYW>Z%=+2S5A&a`+q=wKCGPLDch1lYbPe# zol#UfUbZc0f1*8PORz8{+aULM$fWfVHKhqT{Tq-_eNM9dicGZ+^B%hZ!(GA7Zoseu zFr1mrWgQ=jwx`}>R;(1k`9FX0flN+`)sbP?2}O10F$n?_~R1G1TBP&Pr1@`crUOEq~+q|c*K5Jus zyaL(3*ke0=lef}w7`T0sQiQd49?Hz;+q*ptG7He&pVdM+JF=-ewSC*1&1$HZz+o8JdAEQNNWkYPWfp!A@48F7E9 zPIr3e`Tp>Vg=HqwJpph}-J>(w+9teU3qp9BV&j3WlSo{drT6LnA42LRdu^Il;x0DF`xh z)=4E0#u0g!1qv~?h4I+7^N^B{#@barR*AO-r0*K0|UHHzu=Ae4D9 zuh&-8b&(m`_DF|8*A--*PNXT_>n2Q~DAdw1(P8+0qMk`@->PrFxaXbO^KOn{{2ht=kK+03L;VHWIklpUoS$UEUyv?Nkp4oa{k(MH zr}u4wQIUpqIm^^s6_!&T3!PR#XXBt#GDUxro_VTAT6IfAQpDf4x8Je%74( zTgP$4U+H6So;Y>Cub$^qy~WlP)||(p$f(#STk6g&&2)5^KHI}I-OZ)bF$-wDW_9k5EdD76V5~@ z;4|_0nRaG6?D*~EF%x+L*fSSi8F%TUX~=XE9u=QPkNyE(@twIS06qeZ)+771E6fvMZy5 zi%wvuBN4AUo9;rXrp!!d$1{a7L*!f_avpSjJ`lOU0}*=Wd=2MB7Uo+QGDI#yQ5W-w z%%OLjUthN=oT{dSPY|eGC0rO#V7Esq4P5Ij%>6}N;O*(V-f0{MQ-nZa-W=(ED zNjGxVWayfq4Tm!$VsQ7rqXd?I%Uh6 zah+U55cSi+%+IrTxuz$9!ZPUhDWLE)QE?m|Ddl?Na6~Ep6tBtsL~;wPXBZRDqRizy z6Nl5B!E<1OOwlpH1mPh}>-PP$OIXf#$Rsae;7L>mzplm9&Vn*K{z&&Up9kA7fWsHT z_DgVconp~FlW13VPBV@ZGCam%BRPfE%ZzHW^;Eg_3Qu)0rTSG+O%@!PqB^LKrj9B^ z_lmBY=xm9o^^*mv%_8|3JW~ChvY9&7a{VrU(0PW^#J&b%SAwzELF^kK_Ox_*?etib zI6DIG8SGC-U|M9o$#7kTqTb?hoks)Hw}C4;1JVSpVcR?>j7~VHDHTDO98usZs)IBV z?*IuhzEp0#3nbnH5{{`6EOTn7OKD1)Um%!QWWCRjSc9TI;E`BNJ=ljpg5*t1AVD(F z)}p)fsS@{Rp679E^){OJ*s_2g}@ zPHZH{`j&D19m@Tl=Xx6DnjCa)Z>eL5>mUc9GtG>slbH!!WI8P-8zw&Gs(;n|{V7u60k2g$@re^VEH8eSkhjdAU)U8f?iS#DA?zg>FHmeZTLHD|eAqf#vI{r)a{ZxHY zCK9xMH0bJHHn1_QwzjbUwRhIH|E;nb+W%(9Kf&f61Cvo_{8PsUfS=WHkXvqHq1DK~ zJGoo`Rw#38Qd(-a{;R0ZY7F+TGUT&@W?!U@|M*PqP+A}PWIk~laM1)B-WFUm1sBa6 zBs2#lk)#*o79cdU|6BIHV=h~f)tnKs9ZGM(6SAHXvONeP2cDV8!~{LTMwI~Bh+VZ@ zKhdBct7@YDBC8UF#}w5;bltC4WYp~d>RN(}R-kT2LPt77(wcykN`<5|ByEzs4eoNO z{XdSwws?Q9DG$9r<)IH99NJsSGY)sg=e6TGTt_)<4-U!6g{C+R`@o58&s~%`$s|W2 z9h$HcBJZX88L~D)^xZ_)RqN%72zLR4yMl?`z+eYLhJ8{48<4b5YUpJj&b@fXS@uZ{ z(LL=WyRU5Z6OUJ<*y_me>V%>?^LS-Zyt)9dEJJvaWgKCeGbd^)6DC~;L%-%)Wo@W* zqHK4_ZVzasD`b}q*>#UayZ>0d^oD!0?Qt%CNPOS^M~--BZoXrq%v`G*lii*uYA;@P z`BZk@Av>~{kqN6FrXg{WU1f%fwM{?)Oh6&HErJOsCiKRqJCfuxkR)&X=|#n#Ueqf-6R);wpLo7~ z;yK4Txj9w|vjx3SW^djWETgud4{QNh)?mUGgl&OaCYkaZ6sx4nGg*HfdnoI+MI9bd z9Yoi4((xwQ7u8dA-Ou-Mg%EvVKlTCZrLZ4Sgb;1gA#U4Pl!m)}h?`q%^<%c84EOiv zZN+N3wdTIC6{`(xtx0S}eOaL=s})h0D$+qFd9RIGmMj|p4Gn|_V$jea_@G)=xjD90 zE3pPMB@IC-LwO~wpq-^*P|^y6oux4K(SwLcL`qYQMr5NcKbESlJ6k5cTWuDA(X$`u z84lg=4|+y`o-Z5`+?R;iI!q)#@i@r?$D*CDAU{_+mZ#)d2QZe(QRab;-W+9VWU@DB zh@~KVbE*-PY&+^$SyPr?$C3wOKSYt4YTLM?kx^jbAn5jBut0`Zvg1EH20MSjVCOa% zV;LO(6|Y)iXytv!1fU%25XQoxDDyC$h18_f!@+{dNvR|e_iq_qQOzC8@~G;I>dUH0 zKXxV)lXgV)5MB4<(_OqD0p5=Un@557G3b2^iADP*=Q30d_1wL>u*e$A*sj3+<9N1H z%P_}-ZIjC|gJfIO2BPaa{=}~>SIJa}>My!p*WXX%52eSPCV+#Zq2pt~K_xhFZka<8 zoCJwkH6byPArVJWlf00aY#fO&xH=uF1QEWOz{#)NWY&4eCg)_5+J3NfxrQk~;#lZ- zDv+2~W4DOdWHLJx-J(9PE;-3j=pLb*QU*Wo?d#d7ZYEO76`7hcm&<@!)V4I2`LpPgEcwvE|`E9GLyg9r1B^ zwd(zX66*xU;fW~oB%Z^?G-*5=94POCkcz1T zH-g+~QwLp9GpY8KI3+Yb|wu!NGA=tSHI=>j~%z=C9lpeG09EvuWO1;d$%eJjw?v3u>dmU`XeDJ*hHsgBO z49uL3wuQ}T5Xa2f(r<9E*H6irv#GToj@fWI)(y;N+=w!7;%&wfn%KA*He-pQ#72<4 zRgrE?_yb+$b?PWRR&xt#EPy&wL0`9dxg2_x$b#d-s5yd%@l!*la%=1m)VNrrpPQ zydPyg!1FknW=$4@$H@k>CiTT`i`rYK4`o8nB7dP%Khbv=U9au4r3PLegw`K|E+2;0 zmk@QPw;*`Le(A9V!K38uXv?(-Q@AY%9wYCUZovMm|7@~<^HRpa3!dp!s}i|VMOcN6v3>CDPo4tbPs5%( z1K+vakrjE)@oG8xLS+AB`-is~_D0UnGMlj+Wj@E-jMPTX&%yemd4?MfT56Uwrsp zj@`#{tW^y8w@}pEJn}1P7x*0@ztT__m>^$Y_Qv9H4LK%h;*N+Iewh^(>b`>9%M7cScZm!{BoGvLx3xHL^?QJcl0oiJE3p|W`_+V%0GTQ`1q ztpk^0YdeNZ3lz0IkIM@bmmPr13x;qZ>7O81n2VaiujMkiVTfNj@O7vwNvcj55Vc1p z0x=gpErCxfXnsfF(;E1!OXKsEz45TSUmGc~+AwC?;{Kg@W;W6&W@j+7(NGjamyHA;C?Z%|m z0Y!D>rM8%w=T49sSuSW|oga}}nC5wgJrejg_bdJ2>_Vk8Q0W4F?+#Sdg(g#(`IQOuv7}&N5N7*xG%@lygn~({mbX8o_T=GDWbFVc!cswtTKjB ze-yPZk5G5&z6JoH?uOh~n0hQCnTxL1H3m8o)cI~wtIGyLqA}=d5F|Pn=Kk_6xlTg- z3i2wN*J#fDt;?Q^Ux`<3zCs4n(IY6ZhA^!TMVZ5Rt?o;;x*xQ6?GT?le2bl5c^s-NsE1)Y#xrw*(Kdl0mFFmyzcDpl4Y2mzeC zM08oX_@t`GBV?q1;LtZd?l9*benC!7v2`eug#`e+pykorBt~AbenB0vC-RSZc2JMUZReeq- zLQh%yN1oqA`*)`OJAq7SwtvY<6(mV?82wxK8H?_hVtXRkj)S#HV0$uL@~CuD*`A+D zdKC7Lqbr_oO<}Aai~FbYthS}0;WV(?)?jEDL>c{HC=ag zSrX8o1q~es4Niv!Jr8vrZeT+R#kQ zi4c816J2s0Um~iXc4qVh=>J6M_ax|lHuUf8771pj7Fbmbg_BX#DLe|P-Of)13MO|u z53*aNDpy&TFKS}d&-H_e>Sw6B0v!ZQAgB`(xm?C+K<0Gl{0ty-CQ*C5HNTWB-UbWE zNNi8CTy6PRj&4W3brxf%8fBi%vy($p3g>{G978FE`ZQHl#E0m*swNHf0eEfZ7)Wo9q4Tm+U7d}BTNYMzULO=TulezA2q zlk^oR>PlYHBdDaWf}}?nlyq3O$1ouWk)(C{Mq!Y%>NAoF3fXYX)sXQu(B8F>@jMvi zh{Gu7!6=i0Nf4D^!vPVw_E6zNsiDH_p=VP=g{1RQUvjsyGwXCT?YA$g zGK)yWg`Z%=WnyoDHgANEZh|&%hKY6biIbYN*ErZig&up$7ZqE#FiGBuqHg0QnOcH& zJ0xjx30jadhWwfcQ8TKRYdU7vG2NMn0%YQ0*+@(j#rVaTnf5z?&z;cxUBG9dGCl<* z*4+%Bdr;K9JU+8>7?u%r+FEkbt|${tarE6%hua>pC(G0x_NXo07Ar4x;N)BU$%B z^7lih4?yyZ5zTna)lT^hI#+v$dvq(3Z#~Gg{}9T2nAiRWx)O52#K z{j_^sr*xT#eFS=Y6xw(UdRq#;O>#uflMy{fy6$*Nqcv~Er{LA@_r29%@^(eZJ9Q=^Qtr@-0MVB#5Y z_AEHN#KGC6AgfUi@{=9EEP2FX@|9zspdxEIzLq zWl~l;CQS9a`8j1QQ}q+M9zQ*07bdR*lQ*F4H-X72Hzr;gnvw$REyl*%xc?oVjiz+u zbv4*%YG~v&=u?(d^&=bN>S#u!3)MQC&i;t%r+eMR_tOcR{Is2myLZ9ed*J4MaJL4b z(QS^<=yu1eI}jR`y1Tr^Mb-z5#Sc;DM?8z`sh?U47S|i{Q{)(>u%vqYw228;Wz#I8 zfdLT{b**`w`ibrpUH2Q>vTn%7VC@rd@hMpQjQY6LW|3r<()1*~RE~-7ec0~qbB5R# zDC$cucel={yQ?o*B~cUVUN<3sT?9qn!9*Q(+>%cHbncbP@O}jZzXmto0KxTecgVO! zd(aW;WX3J^3K=&q7V`_OZyAfGZf4sUnSzDRx>X7NA^&utqA1^f@2_@TA)8Qp(Ti{}?fi3%LYYY3| za%)Tb-zux2{cm>s8$18TVyXCg{uQ^nyOI6h#-fAW35MJvYb*O&as&OXQF>!i?$9kO zw;Dct_sTP*;Y$w#mXQC$$4mY6<PT7J>hBDxwu8RfL#n$Vrtn$-miOMdN8En- z`|)O2u$jC2%o`7mci!Z@%UzjfcSD&Sc+HNan(YY9nmX_DzgZmVy-R8{tvb0EQ(6|e z20wLlMKYbB>CVt!7ifBSXnKqzk24l|oIx~?Q&CeMr!e2zgK4@e%FO09J%?(#8#Fz~ zkR2v{(jb>OirS?%J1DCSku6rWWhU~1qVFxbu3Be5Q&9YV1`9{_1P^;byWPP<4}>Es z`&igJ{PtuR?2V#wcnnfI{N@4!Q#fXB6wXSupi|4|kks)AYDD*rGjO+)? zJ~y4eyFC5p74~cZHyZYdPQw`*`=h84JQ}GJoelsRCQoze6~9LOCAPDTQ`QG}NCUe*WMFFp3MKA5~6Ek3*Pw6hx@$BlN5 z#`}Bx@cphoeBZ$_*R#+%gmHN&KJPG|%NBHL!{Okvg&nn-$g2n4k=4TI>n+p)1isZ?9Vt^eBwC=&EDSmG)CR&DD4cM zy43d0XM#FY+dC8Lf}D>pQncv0sv0xVr_EHKuP$>@E{}2+R9y`no()x>Lsgv`si?G7 zy(e8|QlqNv?MKgLsy+{;ozJT}wf*P?P_@bJN5j(V)QNM#pmq_FVW2@&N0a_Sx(jh3 z2)YQGzZe9~0YT0#1l_XN?n0!v+GI<-OBg_QDw!{naa52>|N;bSu zEjJU6!jCLvl7prkg^p=LOKt^)Sur(uu;a zV$sH~f#yggV%@-WawG1)iPy<2I-qqkbTZ4(fL4(9Sj498^)P5GlXlO<_Y-x=MQxz- z{Y=zfrrWJkM=#&v>cHFr#%~3qw}J87!T8H*##h9mEocw@70>XiBx2pcaK95p-Noac zMR8vU+_Mbf9`uwe)h1UBQAl(79TU@$i4G>UNlYdD64gO;-EVe^ucHi(qIyWX*1Lh- zJz(WtV7CZinPW;x95YGSl*i=!kuvwZu7VuvKE~huDDwfHztMCAWij|0ZD<4~Vpk-{ z?1AXntMcimE2G0%rtS@jL8

gV6Ou(BQ++^%A%f=h$+qf%88@$&X&3o8Ovw}A|4EpVWge!a!0ir;i>;@a{dgMpKf~LP4b&Yz z3;VIbkUI=|n*x!}blO*^Y+259!d;#E>D(*(-9$Dd=(ch7`j&&O=fKAEVCx02F+Ndz$e`c3#}I!W-p*=)j3jvo^I_Qt zT1QSg_sU8!lbDI>FS>3boDiW_TXf8355EE&UWJBV0}d;J!_0INM67oiFYlqu_jz8%Q1`J0yo@pAJ_s-MX`H}>i4^Hxq^nFu zE46VY>*D-!WkXwCw?#{_;!xb(J-EADaVTDby99SA zUNm@dcZcHI;skeVad)?y=e=Lf9|&i&&Rk=TeRBujbV+;Df^HVEKZh?WrK1P*|J`)U zbM_v-sB<4KMU3tsy#5u#0o1Be7v=Ks1irv+?t;GLH7Zu^?)HuZ=R6i;+N(0WjdU~6 zsL!_R#qw18_5!=K{bj4QPHD5R#t);T2`0d~&AhRFf9vrsy?s+R9)BIrBrZF?eWI-g zw2TtL-|#52fHAJ~8VLye=P*x{IkEZ4p!v@i-k$CU33XrS6qv8H{-AzC3b3f{)VpY0 zhE6q5Xz(Zvp2w@Yv25PLARye~5#D1**3%mZ#8cY5;DVP;5C_-;-k|G&c(!o%a|? z)S9UXCi65304eYIFs`%5I)y8P{WY)r!dOpwPG<4tDa)_1zE1BbNIg$PH|T^X(bG&5 zqyh`kkuy45TRtfBe!`<;pW$wes;uk zzv&ok4UD6I*!V5tXxQ5`bfdUFwrmcu+PQ>8PHTw`NUzI4Ce%!1cmipz@IZej(&?=c zUNS>7PJ>KPFw_!6>A69`GV?>txMq<7Cq$h2q#-BTTe#|=O+~4kprWZz<-$q?-i9k& zg*oY7s)ju%!TP)oz#T3dKb|iSZNgv0B4`f3Tm4YjGHP7K|Lt^;V}($rd7JaA3a2eb ztPXKM!vX7(wCiI1a6yB%#Tk8JCQ{qsO@NATrLv!xE`~q;RueP95*j~#i}SY*?lveU)_l6h^c?q|Kh={*#z5)f)pZi+$GDMuXY$=>CtD z#O97_;5K~7n&GISCkNjmrW*|j>g=0+#mXm*cPbB1ZPvqC#)-guY7DTekj%7z{-%R3 zeON+|kMR%Sb260tI4tU9JTV&DaQ&{sCPf)iaOH)gLEpL$3$IDviYOw-quE8R`N%Xs zr3MsL2?J6MY8{^d4&f7=D7kgv9`Oc>O#Tr_cj1`L^=c-sau~`Q-|7pQA7gQ~h(<2D zNF-|ayRV^G-0gvcvomi*r!L*4=(14@!Y%eavZa3@xiK95KAux6l7Yf!Z?C8!iYnGe zC==(f{;87J>ggxBGjzVxs~l8u+g1g@W0U8h#95}gC(h=y{HC|g6|q9wO?=z;@r0Go z4NGb5lm?ypZqI+OW$W<;D*mdb*uY6Pl$pnuNNPcaugT&p?olKS}j8=U_(rq87!jTtkDU)I16N_n-E!GLS##y)4J^ zCx|cMp&75io1fs^jpa~x)_j^?x#Gx_eg+QD-$6iezujaOU}6Vc!QK>*zrsiVWJ3nL z!wn4+uJxo(u{;*Eygvb2cXh`~EMwaQg$jHBNK9S!nVNGi9USpDvJzXh;7(7f^1Y>) z%St4IKACTS0ecfbWGm-f_)QD7MSiMp=^6+sfA2#aS)fdAE2YTEca%S%5A)>ohD#Uz z{6YdxH$-9R$MNUN3gdSHX)?EtsJ+^*>f7P4(2kxtVKgp-ElK=0TdVv_@#(oq$y*y{ zczuPKD?}>&BV%12fq;IYd5CW3{@~wxepB6{*gJk%*@rHa+5fj!Z)?! zq(?h)RoLltYSJ0>mzm0>Qar?&+}^0#3GJwo_*BoV|LtOs0;D~@LV@rlvy399gFB%0 z3C_Wt*1?_C(9{-toNmvzyt+`kI+~4fCFXD-v~t`EhFb zqx_vuLCXV;9E%uXy`PCEORgqxy?4%sTCr^;e_^goK%to^dUu4Wh@-l60{j=d*_A*Z zYhK;mh-d#F{X{MH?5JR_^599$)-m3c4>NxUTGIwP;x-B880muJ-}CAmDr(>lJoomB zg9LKY)owfkXmRIUTHW(6V|h|X!}!B_q4~WLVTTfm zl*tRK(6P;qrXgHnP6CJpf_|$lN;?-5FUfn#kRhh31r^f=?*b&XTnYJz(UsV?&twqo z>tls(N6ku5s!IuS)e6V0yhNQ!EuZu+ygMklNV#(Zy4vLVpcx@se=S`Gz6pA^a0IVE znV>h!p0EB&j2JHStH60`sGMP-&+cr!^;`e0XU_fgc^tgZ?}Zp;vj2^M-bJcw`Y_(< z!Y=g6uR7{nF*TK=5VaCn=&J(j4aM1Rgg_ivBEHk0dGXcxML+oTE@WO~SW9Lb@F4w@ z-TEE64q!f4a~V?JVpW!m_xW$B zria7nKaU8V z4oR`8sAz&X-+_YR=vGvu>CRjQ#>#ZZ=X_r^&F0Rv5ExZOKhTp)&XNZwH#?i7%uN5g ztv^s5e)QAy4kcy%SKHX8k4Ou@Fccf7_GPA2n6%Nxz@E-W-9H|$?|vjab(ZQdQ1TNuW- zm^cjs!VeR;vpw+Fq1L9wg~o0Jo!RO5m}aNG70<1IzI81y3-j0Fc|ne^WqJ<$3J^Pa=vMn&2&1@FWfR{9tQCb`S0`c|Li0qU-J*lVfU5tZP|Hr!dV8^L<>8qO% zA@vy!ec;CLtR(qCThmQxjE=hZ28<4h??vbx=`yp%Xs3M4qn3oqzVHI^YPkCxDty(< zT$EWpm17c8q}WbWMig9I47Os25<{}QA$P%4rtWsnCePwnlD02-)ejOwp`^94$-2{k zMf>5k{>Wyu?QW#ewbcDL5~2}a`1^f=Xke%)rJfwQS*LEQf;wEa;6JM4Oqs9u+wDe< zwS@fE=o5~J=wEwJ(G=vSq|B!S&gNVcYMMo6U(kQ`(AGYRX%2adUk4J+`6bLE$v9j% z%{SYkZdu`I?=O#q?*c@J)QCIySFwIJBhWka>Ki!krT2{?=$#UF7P(o!+XKAK=)>10 z87fHQ4Q;O^eJQMOZb5T>RULpj0>GKim1}S~3d5vWUih+uvolFg)dt<3^-2JjL(4)Btv7KKQG-Yj zC8aorT`;LW*Jkw4*tY~GaRq7`57e*A*Yl(wx_%lZ89jD|o8RR)a1En0Lda;>o=>UB z7%ke9_L+}r0#e%~<}_31yyoryo!WSty8d+FO)iJgJa@?yO?E_~`-d(m*^0FH#xM?m zv2(kY9WF|pzkh}xmCHe@O*!*}2|EDK@jK@YrU-t|SDlQ%g-?VMVZfREDTX>9%wK{8 zRdICgKU(g~CItUevn^2GDeYNb-2JLgrZUfpKuP zcBqfJ6BYr@RcQE+4B2E(C1)v6lH$EcFg5S!-2$L8<3|eL{U<22eycKMDvvw2onn({ zD1j)xfbK$GTcaorK_ewR0a^3h1h@yHsnl$#DmtdF(GaR7f%VpvOva`>|26BceC0u& z{8#jeexPgmj#F9x|0#@;i}y#m=UehIg7j!e)*r5^Xh6aT*pjXR*@wF3%=Q+nG*jhc)49H-eIU7ewvJXAQY-$}PR zQ!|1z1vCgXO6EB6CJUnrnJQNOm54M^vcemw6{K30Tv3`ETOBio-M6b?YR&trs=zF1 z=FCu4qC=o@$1s_moBF6B@xeb~oPDLA#E?bmYmpP^S zFRjM+DTVFg>K)4vKWF(R9gh~8q4ZN@_O8!cK!?Zt!XE664%yNM_qp|LX@?WgP=3_| zOcmVCxDaNv>!y;c^U2<d)K+A)m-$k7? z^plN@=*FPi3{sZS+r5Uf2Vr_T$|RfI!LueyX14pU6yn)y!H&eqvY}s2G)ZJ?fEQC-}(J{`3UwFf%uspX*+$m{(8E4AKK98dZ039$ z`e%~!F90*sVOSHcZ8g6F*&<@noeCi_jWZE~xeL^~4lUUK7i~=z^bKE16GO5d1It2y zkBGrY$n*+HyRxQ|#PVFA4zf7l>F5Wcv)U0E-)V!B5$PY}0?BR^CX|+QU4@_m-HRyf zdZjhWm>I_wOcs&k)keKhlZ=qFMbJk?VW^2+!q&azU0c-D(L?604Tn&uMH(0vNh${5{i5h;1r1*6aSE}<- zZ8VlwJ@;(ENa*H~@A6gp)04_&{+9q($n-sL72JXmXF~ia8c#`4Fd|88kNj))ByLA z_MD1Iaj;BcMmeQ{XGUl7;ZsM0c<#q}Nblml_$>;C7K$7_Vv~bbA+-@%KL6rVQhqzS zjQ{V#_Gpj5ZiI?TR8XKiyP$k6MUuTyJ`FQp;^HN?L3*#ujiE(P?r^0ANwHJVjL*Tl zLEa7NKc;wdP0j2=udAgw%F_09rf9uQt`gq7adNK`JoS+AaAI1 zJTw>@3$<+vpp$<3tJv}uFzV5kliNEcYyE6>aTODBOn78j$g%m9Qh!N&_MUA;-AuUi z-WxtLK{3$uBW82FbaYY?w2R$udkq>8O9j2buDoNfyrQhUXW5Do8T|U4idJbtL%(_6 zzV*z$8F;xtv@n5#r~l4@+R#EN!&CQe5n(W1$o388zcdXR9l#w@CT>0f_!i^KHL~;R zk*lRaQ(eZ33NT`p{k!NbO%`7rc*5d{m)~{t4X}88Peqegoao}#tvW5|_hhS}RrF-U z9&Xu;)eHOByyZ|jQ}eLClfFGn(WskWZQ%|jYpNwV@v0p_A71Ew%c0Lrvu+Vpn2#P# zM^*9sMMMDawCh-%tDVR62I7|xp0hhC&;{Rmw=xG0#;_e8!P24B6psAy7@Ygvav_-w~e$d zqU+mzli=lc@TJfN)BVLsZf^I3^N`LqRwbHw9qZ^?lodxx)Dj_T=BPAWb|ijt4-_&H z3Ba;~{!FW@_k{}UBM3~yZRG(@`0S^>O1{><_!kY#hAGz>7Ld|~?aKJ@cC_T@he*OYrS&qMS*cF6j6zS`)L9qR zlBUkB>ZE0@@@C`lN+?f_aYZ+mj+e*he?703eBqHx?%Sc=-F)w(cHcWyf-n(MKKll>S3^Ou0`$wusTJF5K5a!fV4ONOTaDErek(ADeid9etq zd>WH=-LGX0Lfi(C_ZK-P z)9uv!V>$N_^T+#yuglBhrFuJ|&!XcOCvqz=Dde+-le9-4WXU5PAe`|Kz*Z6-Qf@Z& z9$;$A?-M%-XFHV3zUFAQYeN%EwEAshMklw{jV2Je`jt?T`pX=Qu%fwjo)J3u4*_x$ zlA;8iy3B-Gzjh=|t+^%W4DDY>%xAU#I5NKJ06+}@I|>1R0rBH;qNxi`o#^lnknIy% z-->yIrD_z2W59OTDGP#``kpzX13fm4|)g~5%gHp4H|HBO3)X5TZ(2zA+!`Bp^U zH?}oUAh35yycOp zMI7rpJ?jPY3HL*C*7M`_tM2a0*F*+eK%?FFSFwQfB8qP+7Ma=M0PHyVG}{G5*>(gT zVxXda5en2-(vGz(@V{R81A04_gTnw|oQ@pPUo8U9N_>+m|NJnHdl>bq;^zK0m+EHu ze$HiveENykAw7z!p71QEUE`o&&Y>^fsBrM2=h56XcD|ElY0hFPVHTB9#?+6RJC!+! zqV>9aIt!m~f}0Z26Un`M_Rd=HylSdLO4C|lVSmT9!oVbx?thd#`f2R6pfo?S)U_6l zH&MK$!3wbusdWp_95ISdd5dbYd^}!uu%!SteQk+Ov(s&^e}m ze9(|AaqN@%YlguuJc991k6KuSYY~pfmaMC(XH7C10-Wxp-I+e48fVi4D^^ni!}tQn zEzVy-dybB92>1k}>u{v?QAJw0)TIBrAk|)7FO-C z+u7$Y^X(Nvvo44liDO1n$CB2^jt947ts|#Qm0cdNh zExDwFdzkK??Q3UenNl68<-*BoF%R*~ZG~ZE4}T*I1Cxgw8bNYD>|OI~(t zZ~oPm6T+)C_DMGXR6gvJZsUVA>KC?n<6dznmv58wn~&#&5SS@4yQc)ET2z>N0L?RH zD`KTV^Rwx8@M8dKrU|mYDEfpL%!Js{Cq6Q%pv2d|&CvFCvv4;~?Ng8Qa0}_s`5`|F z<2R8J?jBSN(cgC6@bfRTQ+_B?nHT`1LR)j76j~zs{~B=gT3FDvw2@*wL_8iMDKwTp zw4XP$fKI&j=dyy;-kF;a$Zzi)bt^o(jlys(V;tY8SsEZ7--vd|sa#3=+7wk`g12n_ z|AUuTpP(3Fc}-!s z1{pv9)_QglfBucoOkN~%_k3b;>Wb;1eaod>E3WnGr@Qt-=iY%#@*ypIN~$Vt0@*`^$n8 zcIMGW6bHAum^vMGq#)3zCF(#ZvPS8<-Z|_dkLd~Q))S@k>7)+glF+J88(2R77R%h7 z_3S;}>V{8{XBV5}DJ#F6F>IOw9~l@erM?f?q|eV*aNmj@>Ix~(bFxt1&QET*g^Eb=)H z9d)mQ+Dm8s<38dvVL7~W;oP9$+%!^;+@Z|avL+!%zmN8~+MB1FzmDGA5RblOFV3FM z5xrsfw67=dp!$Ry^yq6AoU;j;Qb(t&=y_&D>IvFD!2WzeQhA40eM6upc#cCaV8n>^ zFa0;Wv(zfrL^M*5VPg6d$U|wt``;AEev>aU%6h_gnhZg^V+bXAzAH@OpnXub(cc$< z>L3gP301Yc!9PNzHz8_pd!i)$e}+X#KK|rscyGptplEVPf}5mqU5U%=EKs{`_o?RqTk`0pUl{nq1CN$N~ zVjC?vxDn{@E!Yf*KUxUU@N0%MsqXEM+p6Sw=zlt6AiSpPD%r_cD%aKH{D(`thp*>A zVB(;@rz&hxX>I?D>*1>f122s|ytim}+?)HublW11FEl3KIfj1lhCT$eb5^R7%0Nrk zO3Pr6I>yt!ceXml4k}BKWAt9Vg+{PH7_O0=)HzUYBcO=x9u)G*!P8bNTFr^s91mUn zyr(I=+xPPwEnzVFdxm#+i05`lZTj|Fv3W}8Lul#m)iK&hd}XI}k+`w!7pipvIrBsWJFt zu2kTwS#J%m=Z>6eMZaU@76jYM0f})3gjT?Do7&n{vIH$1DdjcU{ z$0W6T^!Rb@*a@@(I*jH_(SYYmlnhQazh6L?n4LEcl*{tWn&G0 zRD~TwntNVNmbnr{(ubR6Iq1;E+n{xRD-9EOigfWeKQ zrxskFNn%5cF+L(9hKA10ko?SfyGDH;M2oLc86=u03j7o;2u^#mWqKW;c%4Tm?rDpK zJsvjE_6gWw1q>A&(;YD&&qKeg4viH&ozM|g<${8#gJ;!0HLALtRQND8EEZwZkHkjH zck_fNa^n?iUC=n>HP#IU7omk)nk91^=>3c=!&PBmOr~ZlQSg9g+s>~;n));SWr>qx zNo7{2U7w}hSHmD>Vma4OmQUlGHXuVqTHVFi60N*12QGt$8(25clB|NMfh;-M9ULhAL^xe5oPa#N zt?&A=*`gCccWxxw?A}-2n#D)M`Jdpi@5+cnT)k_`&VIs%!(%r14OFlD%=XO?x*>hY zw}+gd0@a;M6;dnz&lfzPpP=+T$q0#!j98$1Pp(Nu=Gw1ONUsT<#;**bblm2|-1Bn) zexn&%MeoB12YzYqsrfJ+4-2*@>N0-`#*SVoK*@zjQ6Re&huNmi8-eAwVrZXb!ZHkL zXimAW%7n&zd9HkZa|)?e!!iafKqrKDc@j>-O5%B*;=mqrr`f>K#Wg+oELF)wK-eh} zguZ=f2=*p}%zh){&&9~WBS!uiI*t*^UFhrYhVe7>qBG;lx`9{8a*JCNgmVaq7jf}k zM=|AMgWJUKvou7_qD)hWm8M!Cv7}{_U`)Hx`G(j0Ijv7|8~~&p#$@U;QFxTw@-bgYr`QnK>7C^PJcg}OmE%o0!dah##-q_qoXaKU*kBcDS2 z<4(0>pDWe(c!clE-Rx`O?@K1@tDT-5BUeCft!M!~a5>d ztc&q6m*(GOZav`*_@pLV#Y0;qCtD>ZCuu!8-lYQGXojRv<=;l`$EeyCPL(G;^U8$0 z1=LEh7L5|F-KDVb6R&$HA?oBTEvlrTvy>lR(xH?UCGvEL1?{88^F<7W-I!&YXrkvZ zNB2}KL@|*SN;BranMtM>`0p8iEp=y}Y&1o329l3<=30q1`_96eZtR??NZd7qOqC3O zw*VlPS3KAKst4bJ42!Bx6GmErKi7Yo-b*;&VJ`U-oRIxn%D??GROwr1V#l4K4LbQk zP@l@-K`xOIe|5rE@OR(b)H5;vg6qKIhv3@7DXqQgGpJG&Hc<=!>T0RST2L}THYST` z$^F2t7J?uDTZe3{lFIw{0zh_HlVR5;Ct;4@or_{{jzH|vUP87>aDz4ZKJ!9R&{%t@ zGBPJdb-95|3Fp+ZAr4xx4G<%YXAx;W*Vl{QHGL7#i{2Lz?CnZ62D=9x=4RPkPUIrv zYw>8uRUq{)rvh(l+0@sO1RqkFu0HDK*gs`XAImKe%;z{_x61z*XXq_|`>2b>Cb%K# zUaJ6<&fOiV?5NX+x8Dd^*hJtxr?5OdTC$`jUbWwO4QPJl65Bwl^>G40*^)mlO~&=j znU~jWqki%GR(QMn{uf+hm}0lIJ}L9zsyK`@vCu$-S29g)+OzCbps{;}=->_3`bO&D zjrv!N&@8pB=kJ2ogUA)h^TK%Ks%GUg^1?DTuQsQmu_4j#YkWjsMe$O*I9|9z%g=sA z*~C(fOK++PF1@2_Jd->>L#+Ia9F-T9I>4-sMq2#2>;L6@k4bj`J2 z<`z5WGH0f1oG6-0!eu+;?^uSrY+glO_4PCAjH#BPH$=2R>$-)RXm`ole58Y80iHdZ zKlGn(#V-t;uSqdq$kbJH%-5onAyG0NB2O!{11syq?7fvau�bTaB<@cy2M(XtBS^ zzwE^e$5fUtzaz2K@B;5bbB^g}vQ$f;-M;qN0>kNR*@Gi1>n;3)Bi&XT_SgD@jIOUe zEve~ul$*ED|62O!g?6-nuYY!Kgrc9w*-N#xR0G9TD^W+vyQQ({1jDG3S(6HL<5Gv4 z+KO)Ii*E2&-kszR$>nV{1Szti)$!w;5Nq<3q-KYsGBCoO$?gK)UC)Vp;hZ|-e_mW6 zCfb82jso5pPB|z$1TLR>JZk_=or255Mc7{Xns)UactQeN|5yE_W2$lZDB7x~W&YfA zds4U@e9aphN5|lAkQ1l1wT9!Kj`}lH@$j%r3R)oc!Gvl4uj>O+X^muo5}tmZ<&>wE zc?|-;meI9ZKBSOoiWi71(9*lnmIg^@wwDCq$W@&Z#t7w&rwYfE|ET;LETc)Ri=`$L zvR`*mHfssxIek8=H)-kmq*V7r@?~2wwCf7eH&OqxMMGE#s%!}h8|6H-cCxQ>4;5M& zV{P#GhI$(Nyn2aR?1OUJmr%a((TqX4V*qiQd02Xmo$LD-CjTy!n?~l9K)!=g*b`R+ zIZ1Q+B937?;X$@@!-1w1!{@EcIAzW3FP2uun&qK(?c$mx=rxZ{PI*OYf+KIVC?7#5 zHArOA9$W`Ixs?%hLfk!wH_N!0knjojDk256<78mWb8Jaumu(So^3#7FkBB)NvMyhZ zqwuQ(T9|P13d%MHb@kN>*m+ze!O!?@8xtZd(~oZr`N`wbWZ6FQ5pp6yK57?jd|x0i zM;Ijog%Ky!c;|y^y90Luf`u2`>EycByMQ@r2Bvs9f70yO1dO>6qhTG zCvu-O)&)}yqtD6pjIsZnkn54n$N?v=k|6K!NV6h=)qjqUoJQ}t;yW0tYt%NanL>=& zL2@j1X#JkV>f2RzSsZjC=?+!JsTgREYF|uw2BZBg0tEP*5#PZK71yDuFCoOAa(40# z4l~z3aoT+e*Lb>K$P>D4f9@x8=i32={!GkaOgWq?cB{2_W1q% z)FTo&kSqzk-mY1tbO`*2@6Mw8mZvIoUupyTZ(nN1qbRb-Va;PZ>;~WU+Huf!Y|LjS zAhR-x@aM3W2Ordc*P;)O#bk;HIl}yZqqbZ?Y-8Q@LL5KXGJIUc^IS~@uhN9BS zx!AV(FK3`FCt2jG`tfkBym6OKG==IaLJZ<>wiak3^z&)n^;C2((smMiR(6dRGXrqg zd2NOH@#cALiq>bYtK~&%`d$5(M~YWWin2TMbUh6cyLg55ZCW+{?U&LeZTzD@2kqEE z?Uz97$+^QnW;2FMJk&FtP+(y5Zkt|9iesKqi_pNtfb`|TEUh~C<5?U$wPxQsxAWo( z9pi)%@f8zMv3`2X3(pY05!C&$rvIP&qobMIoS$^9ng`5Dqo4l79e^o)vaie6;1aFc zIvnpffETLp+)TJ-N2kY6<~Q-t#JJJ%P{(dWzty-NECTxm_h<2cqL11+)Q9D*G3YaI zXdkfK;qIsCu+7TE)dL-Y&CJ~_b#K;+WJxM||BY6!8SFQJN z8i?QyA(Tw3Zk|Zt#S8iwMS+OSFWtf3;e{qr{_7bd0Y4~BuE=BWqsspr#E7r|35>}#unTA+Q_n_DTtsrlt$ zY-HD$ozLO@#SBLPvhSDNXkz#bv3%fg44)n&b+-d!-ITUm8~u!=&NENG>0KPqqx-RS zztL!s*2}_dieMXwwFwMaE!>JO<a1g)n_%-ZLLYSt(|9jH=AxiGDLd;k?iEZZ2L%;Au zqDzP%+OuG|!wZM`a8D5>_*-%QL`sP8h4Q>mWTwjlajp{67!bd3;tE!=W^;87Za=Y z;`Lh|L+WTUn(6}fFos2mv9pmAac{3*x&+ez!O2=V&k|b?m;^oq{A|JV)z0=%%JI@x zm^Wl&KqRez0pb(p^2jji7rXDuIpM|*pB96*`!{6VC1yqKzMbj51RFzeIxDK(Dui7B zt@zqKJ~V?q@0`AG9I&DR9g($yu)gPu(!#=qA7IVns| zjYT4y$J+Lap2_{uIJc;W>tkwtIbVS#Z}gQwvXjmaf!Af`w2S6?BRtOkJPQ#g-)W%X z-U@qr-TBy$O%MqJalJ86xGM}`vge&Cv)zlkWs{oE&tWg(n8*h8Wh^;hae*e9H6w-JuM5gex@B_}nryqsSz8XwsH z(t95t*x{P)pE^`t!?=0($LbuQ&=pazu}_<_@?eifV`00^MO99bMi@!vjX681>uVJFQD4dk#6M^4uoPIRl{C_*Y5y|*(&tL01HSJ3QkR;GuBWW1hYqkT9i1yoSI#lBslTWFb zR>LxqMrxGgsy08~pZ6Zz_xi)GJZ0s_?J>I<3-gc`q3m&twU`PB&k2xNeMc5~MY#V{Uyw$)$v8{9NUfdf$vj{GjIL{bIvjyX^$uHt`CoGs0SJvY2AHJT-F$d2M|#u#0M z_86LLzr>dS>C(y+Mjfq2Ji*%ww_4`PX;fuC#K6_yL%gv5(Tu~vm&-#jD{ppthK3d^ zhV!6;uU8EB`3$WeNBvRrmG@-BY^sLT+=jz=I*XtXKPioJIu7Z5iKb_^RjpfXRXJB? z#uCH6a6yT@E$?tjC2Cpmgv zI2vuwYGXG|k2FJBD8Fnnjv{fiNeLj&O*1=T(@H5eVK1n6lO?n*k{f4r`TVFn+95US zD;{V>xoa=4Pw9z2QV(hPrm=x7bNXAUP3ee*S-veMpCYO#@IA}AI#upVvtK#IB!+DT zOx~ye@0W@a_h4@qMcDX!n8?|(amve(n!ZWTv#eFRLJp6H!f@eq0!WLZzR_}S3P&f5 zd2NFUX-KP#*&y7&u|HE;=5WrBpv|E|!Mft3k?H zlb#88-~5AjMNP|b^knVszM0nb91X;vj^&y?u*Zd0})}qP(>{wdRA{{n969XJ(aobat$rz$>`;-CDmUOyJdhA&|Kk1rWIdwa4 zENMLNe!TZ~I&pNCpgNns-m|2fm5FsGtD zYMk>o%J6WJ!VY}RVhnEwZM}Zq4jR3jxiFy1`(VwTtUO3H)R!!v8w&SdC=is zp6V_4G??sxTQ9azVYRu!9@GQB1D6rmvLV0b;;3-ogr_Ko#Q<^i``lfCL(!J|MBs3z z$%h&?I9UAP${4ea2umT}0S#>vFSk$O*Cw#oLMYjcB&AAU=CrWr1K%!tf|IKshpIHO zi=%XT`T%=BUISl5*=UdbD2c!CXm1+Hm!wISq)8a0NtMh}%x)D;T&Wh~wHR@wpqXm< zdubo?sSHIA9Lp3>;u@CBs?VhqC`UXDU5S@Sv4Dh+Vw8_VcT0}s*B4XrBN;6oRYC$Q z-som^ft+cVEP}V6$cJEyYGER{f^M244=g*jEYv*x(v6_63o0TwK)Mru=|}f?TU?{R z$Y`eou%j)I1Y^n3G3vC;lTv%W_B@2aC}RVPRZh-j^3NrH+Z>e9gji{`%z@LhdbcM` z_wg?HvDtX{!RbLJtR9ZLC+p#mO}2)8_P1`>ljW~3q!y}Kwpr~Bj}5%p$gJ8Ca=_DQ z+cv>RcTy7^i&yIFQqPdhj^54nEl<3c!}Z_^Mk`7wjg&)3@i~%}yy2oAM_s{OTLiT~ zdvqszP*b3X8iI|Ifnhy~zyH@wuxIkihs+y^&i9C^qQ>c>!a>;m*}`S*2G0@&zJoNe zaf3@N@KxDTbU@DwXH!9SfcIL9!$$fKuHE#q=XQk5QVa43$$fDHxBMx3Om4X6U0gR+ zA;+h&)rJ7?_-=e7v65AQddb;AZ?a*@^_Mw6^^&)PUd@E^>S-7}u|b<5By4j@9ygrJ zHicKv@sPy>H}C6!)jV5k>P2zxjg-6&T}Kc-04TpaQ+> zizQ&_q%1i*eu;rZL_(9){acHgN2+T2<&Ux#@%~w1xCg(O1Y4SXf5E3v<)^@Wy2UUg zjTE0+6=Uy}D844zH>Di)Guqigj}MjE+Co<%snb#f*XKsHHE;$=fkv*1wU^nbQ;zc7 zD~a_vgbv~(huyPnE94l~9}oz>H*Zhy(Q7AnpY00n z9rLDHSyE-x#vkWtNauFXIqiya^4RyVkp`X`OiHo8Y0)gC48U^v1bkgi+k_0FD#!Bv#)};mY6`(`f#Csx!EcLPxcemCOfN?b z(Y0)q%JF@sI)}X5#_N`A{k^EUuKT)^gH8qQrz0yibTz-0BZF0H{4F{Xg z63bJ7--Sn>5A{LS6ZOg)@s9LiBB6>EN~Ll6+)KHpSL_$peCaIKqv`hX?#_@sme2MH z8)Ui$+1Ua}b`hBi>fls!|trcv8z=lw{B16^hk2 zqiChcc0TcCZm*z|-uue&qqkZE&3PGO320o=fz)q#l8ZL@*O&=!TVgE-o8J>6%_geg z`o|756f;aJSr-II&N@EYNn-I3yn6cKG{e;B^ zo=h{gMfXzH<}Wm0_tMPEABIr8ii|6ZTZ?L{?}*xg**j{(-yBgG9U|_WFc=*$7~%26 zu56^v{teBWF}B%7d2VT}U_leB;+9=;0Vy=k7z`cV{aj zjSN!TZ z(7a}NIq#Qw^|>yBinqDfl^nIy?|#qvr080w3ZmZVxi&C#&!Oe8eLE5t%_4)blI_t( zv`U+Vfm+A@8_M)9^VP?5tEmtrI6RRQ0Yo=d42g`0=%2>Nm3~t5qlc=`B~qa~yubp2q==RH z>*}~h6(Yfg1qn?@o@JRpY=N(Y<^ssB!bC)Z2(BbY-&IFSX31>k?Ig}^J+<10V3>2* zdh8rPlj?`kW^G)tV=nyBf;wPxPxfdc7y*w~92rhV*$gnC9u*8Of35(`MNMh-47ALF zc34)kVwK#BHRBTy1<;4dFeX($G|WdpUt^Tc(XGsWD47 zqfpvI_p#JCXk9ziP~qpAJjqhwk@&i6rIbmLVj4rFgYLe_;ye(4*y=Atz5SF4FS5Rx(zI&4Bdh6rvaVWZoqk9hV=%s^_ENBR8MwF{tW85) zQI(zYo=hdXdW$@I=3M0A9)ElVM{GKZx>7{UKmPbCAZF|MIM;| zK3;a~MxfNE9%i>xEv^kOQP&<$_WLP`tF9xHiCVcQxC!#T8JfEV@|_9!2G48Q67g1p z>iyj@mYEw)F)J!7b8hAGy$wa(F3Q&*_Pzu1wKeP=^@fJ|l5iAOlnUaar~@T)9cwIW zCZxORcLImIpy9iL!z|zs+=)EA?{lKC(z%B-a4$Z8pUA-8-r(Z>U|?@cgNt!mWMZNr zlh4YUS&D&t@ol1O)jH{PRH%1P-LH%1JL@cJNldkDPFX4K_8{;P|#`p_BY&d419 zmF)J)MIV&(m>bxFvDA5(GyDk3d{kt3v=@dw28Ksl48x*aia@m|b-ym!TV+YSN&WKq z?5bg0sHgEbv_1#AoC~c#0j3ti| zN4Y7}I#Qpv@eDHao}^se`{4Tnu=pYPUIM;z%w@hWh-XtDDpwLShev0z7s^1}FH+Z`G8)))d=;u3Vayc~lVIYOKByjZ+ zQh57?=Y*CPIV-p(SE9`CMNJOz&rpFThgi~N+?H2TnxXEiwztlFeG@-yBUM9bUgsT! z?ClTG_K(ovPtf*iXgd*5b8+CR1lsnjZy=55o?XJVy#{5j6}8RJE`hdf<|-;;VsB06 z56b##lNicGGq&O=st(kBlQ=}GFY{k8@iX-M3z%5XR2-a6(EXE%JItNl?(7;)lrFDy ze&s0qhN6BKQCQ@e`9FZdB1>kTg&yoDs?dXKDia*2vQ= zz4DOhPayOcbpJOHDrm;s3fnFsyNr12z*Sp>@hoxU%DILH#+8Er52}jgtN3r*Sz@4q#gv<+*8U~k7a&o%UR+=GF)=1tiaMYR)~(mxn@8#JZO zgMt4^iB?^YARDDomMdSDl|q2Juline=ltlP7>a_n1#a7ck@moC` z7`4RNfpfPb%IqL=*VyYBbp&^fE%l7zw0@zkG-*R~R3a~5&t}UWP+aQpA{)i(_Ud>f zvlBSn8O-bg4m*LvTwBO?C}w{W-WIaGGgko)3!Pm#hr6N7&bj_JWMDdhil2X|e;$R6OXFdS|Qb7K8p1D*d;r-mq3rIo&!)qgo17%fMGTP^|!} zO;X9kP!^mvsBW4{He^Ri{k^`rS+42pY=vzlN3|P@+EYZ;&(n1WsdY>kBpgv38udRJ#r#_l^QOdN3yzE{;w+DEu0=h{=x8DXv+`bE3Eyws9 z^K!xQw<34SQcuosFO=C^WZ0jwv^N;GHDxJIy-X8*$m*Z!4Vu(}`Yd%n3YRMFmiB?R z_k|8q&~_ha`^mr-)$`a@)mrq`iqrG4Vf9v-E3xP1ZqeA6YkNPG*-zBAzeVHz(6+5D z8kv8Jax95z-zfZ#>UU5!Pxih^D5_d#ov~xp@1UMLb}Fa4qI1HsEdUXLr;Zb8fS zJFZ#DRw4HJ-#>@`e(TLO914pnR1t1|Gf4Qz}4n??NLp}p)4nLzpjH2)6Rv-^T~FA@|ggz z*-6)F%MfRIPD<~UnJt+JCzIj0b+pOtG_h(1< zhr+}3zHa`Bk9LlM+)ji>PJ-OVGNq(149wfQD6_EVy`{4+-D^?$VqCQtwC~N|oL?;t zzm!%uN4f9(Qf_i~Hd-q5rd>-?O3r$$r=;yCYa4QRJM%Q8*VxohPC&-b;?04;02*N{-Zr z$wsm+#8j%3jHvr@kh+OE$=7FP+fT+#n$%Ip41`?W1wj2m@OcqXzZj_Bo6V-UhaY>{ z6!-As9Ep`gVTCh=qkajBnku56^icmdP*47YP^SA0i%FLKZ)a0|&yRq@?R5C|( zL|L|np_+3JVS6dCy$q~f4s53Z+t;($E=pCcKBniHjn=iPej}CKb@a~MI#%!H7- z8c1CO4rTzUYrUYCj|jR0(BAPJH4H$zM~Pp@k-8p5-5?_69}RdTkg|C+V3hMTbY8d2 z+FJ@$8y27Pb|!U@P2W?vn!MjQqR#u3HvzGm!Nn~=Y$l^4*KX{|b_aKUyv6*3(wW`Z zH$7_B)gMneO3d#jDx6z6Yqz1y+eOxoU17CYl?ENqUUt5Y|qug?b!i2gX2e(Ppk-Bf=8S?3NZPZUDUnD)h?E#?mAb5BP zXw8PJz1sDIsU+)wxqs5vuq!{jP0tXo0z284I1h8i9zmIpii{2NTfnz&g?+a_*K*4*dIh0A_F#h%GKFnPbS^_O8YAI}hSnU90(Ibd)uxPHRxrSh)n z?pg1d`>>YsQs+sI^E?zaU&Q$-|9mds{FEh^$!6n3d5)1uFN%^Bb>D_Nppd2NV0vHG z9Oh%BQwOOKGEW1UXQ1=PzAj?b##_0w2w1hTv~q_{){AoGj*0Z8BmXzhEN|i+ zzJ+Fa+lvr{JU0KeD^F<$0{(L69o`)8qNw-8<{050U4`ZtVKLxmO%j*6OcX`b{V0Z( zs5(&hRdYP~@bXdAQ6_*l!BZzvp@!H(7WaLi{Q-FV5NI!f#r+z7bH3zEFGHDMiA>M-nEo0}&$h%g z^U`ri&6#N4)i#WtSTKSkUnCooQ4>8#)uVkJuXUbGPYxfH8lA#9#YQQx~(vaPJDE}bc4s^6jN z`6$wLzu&|Hn0!_?lOkh1sCt;*uWjBCw*o)F3j7Fue}WZQ4J&}-{rn^R(t+dsHazr< zTY)twWo^I;czMfp87p8ZZ^`0>C?|WG_;Xc->riZ}lqhoYs`0I?mDK4WhkC&O1qDAt zzrTQj^`Kx`V1@8kfvc}U$iCs&xv<#zl{4`h%KTkqqKg+h{{betSd5+P)5>&8vr%xP z`W>>Gq@InsuUapi?3wC!*m`!=zIKS!Kf&r>;OcL%TCfG1-BpzxpH8NdUA@^|J##Lo zbthCea6jZ7s^bR^Tq~u54kGnfGsb5_^yRAvyR@`AD zSN85ix4rTY4%TT{4G8oXacH&iN4CneVP$HWyqb4P)q9fZ`F~QyTXG8Mb7V_K*0h1jeOj`#`vM$cXvWSJUhf<>~JE0MG#{1s| zjo1l|m?N{4BbQWEk#Kh9t+*RX=`6OQKQFWkTG7_LP-b4@luL>tnpKTfWlfA~xjIp> z>UXG);>dVq-8$4nS;!pj4wAZpfjvM{Vc1&}6A%-{P7%i`fuf2S4@)!XU>=~Nv|@7s6=D%hs#XgflXH91iQ;(15D6v&r>({dnR0q5Dv1v9(8us6F( z_V7C2>=dosIo~p;lC$3pW$r1mzub$|yMujp9+0iohEYvCHql#MX*r4>g2}VV){n9k zs`khR>(Z`uFL2faOjLohBsjyWUcSg8$Esd`#jlv;Ds~Ud;+*wDnY~5MZuETY-r(#; zOFp)~bv&Zbq)D%usvmVHVBhI2Z|O1q>3 zAiMt1%7KvGLE+vWv29v;xpOd=-61IIP*HX-d9oV-*}Y^*c5L`D%HbADRyHhgTs;Z( z>^6|1B0H)MvJs@N#SdBRfk5Xl=>2e@GYHXKkHCSaRhZ^oQuj>C5GZMxiboObo8T|z@s#55=rR3BR<4_6!dyD zv~>*hIuxdUZK~>polDPZwQ70wx>Rz@u5S+5qR+MIUD4a$4r`Vg1SXsmI>We9hoiJ( zMWs&mlsW=Rooq>|EWn6zlT1o2ruTJQ)8@pHI6Ol>hzVKh|kVrOr6cwu3TH7TIp%rP9WO?IspeX{-|y zRoa?*_a@w!lr5_-s&iyg2kNsrJMP#H9#!)OvUWqY+z^YWfW-;m=2WnF8d%Jc)=uu~ z4Ig2%O8-g>FN?~ZiJZmLQRW#Ui;KOjgeHN-#TK^`ii$&B`)`Yre1AU`(n+nCJQJEa z3+;b4G&LC^TX1jTB~Vl2>XoTv!@u!ogMClW;ZiymMV%)~$=~<%d`QXWzNb-clo$uo zMVV$*6O(V#r4p{yMct`3l{E!2xh|cREL8i1T*L*y`9d&w5pcd3ICse&$wBRVnn;v5 zQ#j6-ps1-L&XfIfs(|xkOHmq2Jk*CNz&etln$c$OfHD!fUWs~c>b^Z6r+L`(k<9F&Jb`^$%bY7X>(lZ1D@E4*NkmtHbz75&qMq)r z^NwYirOH-yYdVt;Rlh^_*;LEb;ZF5C@?CeH#=ja&T>~a&fT?T2)P!sX^HldlbuX`Z z8vivg+LUmv<9J<=wv%X4nM~)5+Z(?{DRDy$wa(F3NTPhW#4$Ydqe$19IKp-CEDalu|aYCCbPz zpK>%QI|{i{^)tO+*O8iuDWRU-q--1SMW&OfYL0ApQ|i6dbU)O2y%Q$vF4&#BVZvs? zgavX@$;KHY7KmaioO`(4x)-0nPqbUB{ll$bw^mznvusQu%5jB!=})));?ir4L;a~- zELF#RjeGF$-~q7pAlP^aY|Unbq}vDf0NWl*aeHTGyry>uFU7@fxoZ>hqlB7f{NJBFTO#^(ByOGnE?EfI4MROH<1|iS?azGCqc!dR2@w3 z+js^FK@$_GTR+~%djpug2{zsWW^coR;RFOf^_#{C2vtAh1jpmk6R=RGRpv@`XO+dy zJDj(7QRaIhZw>vk&A?kjOS1r2J|!-JX};Q$(y^);*BLRa?b-4%QN?#PCS}`lle*6? zvz+^H*B#tl<2uM48`<{EhcU5>|mf7RcI|aS*4Kew41L`!+m;Y0u0g zJtxbCi5D}@^#1_O{|J5l1kJC8=BEUDR+nV9Ai(;FKhsljmD$Fvap0u6BIiF`^J`G% z+JNRgpR+Ebc}qTrMRie53RBgkY%(eiJk;lBGvTPV=axP2L&nEZZA{(Q@q^(rw7~q& z;PV$Szn)Q=dnv^Rr^5fr`Th-`|6SzUKNbED@NM%{_$V`#R2!JYoF-+<`V&*17eyT@ zR;fCe-dDAAxiV2v@GiAR;!p7W7kK;|JQp-qTpCs64nj0=KjfXW<^JZLv&9PW&)M=P znl*I)(cVO}jb&+69OOu0Z(_D=;6fp9Qa_V0UAlYN$d$f())v^<#QobnYYUi-%1(+I zol5Qb#LGUkgye=;%05J${iDpTjV7lgN7aG4 zZ({arJRc=#o%)B*Z-S&-L5Hm&={8J_={CqD6wK>7^|s}6(_05}4sA0zhvn?%55jRd zZen|DF737`vYjaHv7R;E2GSmD$(k~0Z!D*R>NKg$Ry2ILep@v6c4(XSXzuON+@k_R z7o*u#)o16fSoh$)?b2hANbP>XxZ|@yZW(Tpor+KEVBiehKB~!+R z>!b9ZCT75-JxxAqt3#@4#$>dpvOkCJ_e)2PJAsFtq1|1;LnrW1%TQisV8QqDfuZwa zXIIX{ZYZ;}$iq^Phc4h@sU;qm?PE;ThZ&i5t;;Bw%7@!l2wZ0VecQB>%jI>p&apv+>CtsA}Aq6BQ+Xfd{k($`S>jACG1Cy`AJq=drGQfQ+L z?N|K_Hx#v}D2WnpOshL2 zQDSLK>;G4d7Zk9Kb?e%c%YHv)j&(x}+3&9lQ%vC~<4$;&VK1QC1AJ8h)g(~8C7WEE znM!uf1Vq)h@?Qg}mODK;s=ZKDZxPk*{&_|~wYw!$SyU94iM^C%SCyg81l%a List[Card]: @pytest.mark.parametrize("n_players", [2, 3]) -def test_call_action_sequence(n_players): +def test_call_action_sequence(n_players, n: int = 50): """ - Make sure we never see an action sequence of "raise", "call", "call" in the same - round with only two players. There would be a similar analog for more than two players, - but this should aid in initially finding the bug. + Make sure we never see an action sequence of "raise", "call", "call" when + down to two players """ # Seed the random number generation so things are procedural/reproducable. seed(42) - # example of a bad sequence in a two-handed game in one round + # Example of a bad sequence in a two-handed game in one round bad_seq = ["raise", "call", "call"] # Run some number of random iterations. - for _ in range(200): + for _ in range(n): state, _ = _new_game(n_players=n_players, small_blind=50, big_blind=100) betting_round_dict = collections.defaultdict(list) while state.betting_stage not in {"show_down", "terminal"}: @@ -235,22 +234,22 @@ def test_call_action_sequence(n_players): # Loop through the action history and make sure the bad # sequence has not happened. for i in range(len(no_fold_action_history)): - history_slice = no_fold_action_history[i : i + len(bad_seq)] + history_slice = no_fold_action_history[i: i + len(bad_seq)] assert history_slice != bad_seq state = state.apply_action(random_action) @pytest.mark.parametrize("n_players", [2, 3]) -def test_action_sequence(n_players: int): - """ - Check each round against validated action sequences to ensure the state class is - working correctly. - """ +def test_action_sequence( + n_players: int, + n: int = 50, + action_sequences_path: str = "test/data/action_sequences.pkl" +): + """Ensure action sequences are legal.. """ # Seed the random number generation so things are procedural/reproducable. seed(42) - directory = "research/size_of_problem/action_sequences.pkl" - action_sequences = _load_action_sequences(directory) - for i in range(200): + action_sequences = _load_pkl_file(action_sequences_path) + for i in range(n): state, _ = _new_game(n_players=n_players, small_blind=50, big_blind=100) betting_stage_dict = { @@ -285,14 +284,14 @@ def test_action_sequence(n_players: int): assert action_sequence in possible_sequences -def test_skips(n_players: int = 3): +def test_skips(n_players: int = 3, n: int = 50): """ - Check each round to make sure that skips are mod number of players and appended on - the skipped player's turn + Check each round to make sure that skips are mod number of players and + appended on the skipped player's turn """ # Seed the random number generation so things are procedural/reproducable. seed(42) - for _ in range(500): + for _ in range(n): state, _ = _new_game(n_players=n_players, small_blind=50, big_blind=100) while True: @@ -344,12 +343,16 @@ def test_skips(n_players: int = 3): assert action == "skip" -def test_load_game_state(n_players: int = 3, n: int = 2): +def test_load_game_state( + n_players: int = 3, + n: int = 5, + random_actions_path: str = "test/data/random_action_sequences.pkl" +): # Load a random sample of action sequences - random_actions_path = "research/size_of_problem/random_action_sequences.pkl" - action_sequences = _load_action_sequences(random_actions_path) + action_sequences = _load_pkl_file(random_actions_path) test_action_sequences = np.random.choice(action_sequences, n) # Lookup table that defaults to 0 as the cluster id + # TODO: Not sure how to quiet the mypy typing complaint.. info_set_lut: InfoSetLookupTable = { "pre_flop": collections.defaultdict(lambda: 0), "flop": collections.defaultdict(lambda: 0), @@ -357,7 +360,10 @@ def test_load_game_state(n_players: int = 3, n: int = 2): "river": collections.defaultdict(lambda: 0), } state: ShortDeckPokerState = new_game( - n_players, info_set_lut=info_set_lut, real_time_test=True, public_cards=[] + n_players, + info_set_lut=info_set_lut, + real_time_test=True, + public_cards=[] ) for action_sequence in test_action_sequences: game_action_sequence = action_sequence.copy() @@ -381,13 +387,13 @@ def test_load_game_state(n_players: int = 3, n: int = 2): assert check_action_sequence == action_sequence -def test_public_cards(n_players: int = 3, n: int = 5): - # TODO: Combine this with above test if possible.. - # TODO: Move any needed files within the test folder and condense.. - strategy_dir = "research/test_methodology/test_strategy2/" - strategy_path = "unnormalized_output/offline_strategy_1500.gz" - check = joblib.load(strategy_dir + strategy_path) - histories = np.random.choice(list(check.keys()), n) +def test_public_cards( + n_players: int = 3, + n: int = 5, + strategy_path: str = "test/data/random_offline_strategy.gz" +): + strategy = joblib.load(strategy_path) + histories = np.random.choice(list(strategy.keys()), n) action_sequences = [] public_cards_lst = [] community_card_dict = { @@ -413,6 +419,7 @@ def test_public_cards(n_players: int = 3, n: int = 5): public_cards = np.random.choice(cards_in_deck, n_cards, replace=False) public_cards_lst.append(list(public_cards)) + # TODO: Not sure how to quiet mypy here for typing complaint.. info_set_lut: InfoSetLookupTable = { "pre_flop": collections.defaultdict(lambda: 0), "flop": collections.defaultdict(lambda: 0), From c303e70eca972bf87dc4f7ed30225e27b6aeeb6c Mon Sep 17 00:00:00 2001 From: big-c-note Date: Sat, 13 Jun 2020 20:35:41 -0400 Subject: [PATCH 42/42] refactor and found a new bug in the tests --- pluribus/games/short_deck/agent.py | 1 + pluribus/games/short_deck/state.py | 6 + research/rts/RT.py | 10 +- research/rts/RT_cfr.py | 50 +++++--- .../stat_tests/{bot_test.py => agent_test.py} | 46 ++++--- research/stat_tests/entry_rts_ab.py | 67 ----------- research/stat_tests/rts_ab_test.py | 112 ++++++++++++++++++ test/functional/test_short_deck.py | 13 +- 8 files changed, 199 insertions(+), 106 deletions(-) rename research/stat_tests/{bot_test.py => agent_test.py} (81%) delete mode 100644 research/stat_tests/entry_rts_ab.py create mode 100644 research/stat_tests/rts_ab_test.py diff --git a/pluribus/games/short_deck/agent.py b/pluribus/games/short_deck/agent.py index 92eee131..c93cf4cf 100644 --- a/pluribus/games/short_deck/agent.py +++ b/pluribus/games/short_deck/agent.py @@ -3,6 +3,7 @@ class Agent: + """Agent class can hold a trained strategy and regret""" def __init__(self, regret_path=None): self.strategy = collections.defaultdict( lambda: collections.defaultdict(lambda: 0) diff --git a/pluribus/games/short_deck/state.py b/pluribus/games/short_deck/state.py index fa37c0f5..ecb2d273 100644 --- a/pluribus/games/short_deck/state.py +++ b/pluribus/games/short_deck/state.py @@ -236,6 +236,10 @@ def load_game_state(self, offline_strategy: Dict[str, Dict[str, float]], action_sequence = [a for a in action_sequence if a != 'skip'] if len(action_sequence) == 1: # TODO: Not sure if I need to deepcopy + betting_stage = self.betting_stage + public_cards = self._public_cards + # Must declare the appropriate amount of public cards for RTS.. + assert self._public_information[betting_stage] == public_cards lut = self.info_set_lut self.info_set_lut = {} new_state = copy.deepcopy(self) @@ -540,6 +544,8 @@ def _info_set_builder(self, hole_cards=None, public_cards=None, try: cards_cluster = self.info_set_lut[this_betting_stage][eval_cards] except KeyError: + import ipdb; + ipdb.set_trace() return "default info set, please ensure you load it correctly" # Convert history from a dict of lists to a list of dicts as I'm # paranoid about JSON's lack of care with insertion order. diff --git a/research/rts/RT.py b/research/rts/RT.py index 781dd14b..457ef58d 100644 --- a/research/rts/RT.py +++ b/research/rts/RT.py @@ -10,7 +10,6 @@ public_cards = [Card("ace", "diamonds"), Card("king", "clubs"), Card("jack", "spades"), Card("10", "hearts"), Card("10", "spades")] - # public_cards: List[Card] = [] # Action sequence must be in old form (one list, includes skips) action_sequence = ["raise", "raise", "raise", "call", "call", "raise", "raise", "raise", "call", "call", @@ -21,8 +20,11 @@ 1400, 1, 1, 3, 1, 1, 20 ) save_path = "test_strategy2/unnormalized_output/" - last_regret = {info_set: dict(strategy) for info_set, strategy in agent_output.regret.items()} - joblib.dump(offline_strategy, save_path + 'rts_output3.gz', compress="gzip") - joblib.dump(last_regret, save_path + 'last_regret3.gz', compress="gzip") + last_regret = { + info_set: dict(strategy) + for info_set, strategy in agent_output.regret.items() + } + joblib.dump(offline_strategy, save_path + 'rts_output.gz', compress="gzip") + joblib.dump(last_regret, save_path + 'last_regret.gz', compress="gzip") import ipdb; ipdb.set_trace() diff --git a/research/rts/RT_cfr.py b/research/rts/RT_cfr.py index 4cf69d48..1e1d596e 100644 --- a/research/rts/RT_cfr.py +++ b/research/rts/RT_cfr.py @@ -1,7 +1,7 @@ from __future__ import annotations import collections -from typing import Dict +from typing import Dict, List import joblib from pathlib import Path @@ -13,6 +13,7 @@ from pluribus import utils from pluribus.games.short_deck.state import ShortDeckPokerState, new_game from pluribus.games.short_deck.agent import Agent +from pluribus.poker.card import Card def normalize_strategy(this_info_sets_regret: Dict[str, float]) -> Dict[str, float]: @@ -96,18 +97,20 @@ def cfr(agent: Agent, state: ShortDeckPokerState, i: int, t: int) -> float: # Move regret over to a temporary object and build off that if agent.tmp_regret[Iph] == {}: agent.tmp_regret[Iph] == agent.regret[Iph].copy() - sigma = calculate_strategy(agent.regret, Iph, state) + sigma = calculate_strategy(agent.tmp_regret, Iph, state) try: a = np.random.choice( list(sigma[Iph].keys()), 1, p=list(sigma[Iph].values()), )[0] - except KeyError: p = 1 / len(state.legal_actions) probabilities = np.full(len(state.legal_actions), p) a = np.random.choice(state.legal_actions, p=probabilities) sigma[Iph] = {action: p for action in state.legal_actions} + except: + import ipdb; + ipdb.set_trace() new_state: ShortDeckPokerState = state.apply_action(a) return cfr(agent, new_state, i, t) @@ -115,7 +118,7 @@ def cfr(agent: Agent, state: ShortDeckPokerState, i: int, t: int) -> float: def rts( offline_strategy_path: str, - regret_path: str, + last_regret_path: str, public_cards: list, action_sequence: list, n_iterations: int, @@ -133,7 +136,7 @@ def rts( yaml.dump(config, steam) # TODO: fix the seed # utils.random.seed(36) - agent = Agent(regret_path=regret_path) + agent = Agent(regret_path=last_regret_path) # Load unnormalized strategy to build off offline_strategy = joblib.load(offline_strategy_path) state: ShortDeckPokerState = new_game( @@ -143,8 +146,6 @@ def rts( current_game_state: ShortDeckPokerState = state.load_game_state( offline_strategy, action_sequence ) - # We don't need the offline strategy for search.. - # del offline_strategy for t in trange(1, n_iterations + 1, desc="train iter"): for i in range(n_players): # fixed position i # Deal hole cards based on bayesian updating of hole card probs @@ -158,8 +159,8 @@ def rts( # Add the unnormalized strategy into the original # Right now assumes dump_int is a multiple of n_iterations if t % dump_int == 0: - # offline_strategy = joblib.load(offline_strategy_path) - # Adding the regret back to the regret dict, we'll build off for next RTS + # Adding the regret back to the regret dict, we'll build off for + # next RTS for I in agent.tmp_regret.keys(): if agent.tmp_regret != {}: agent.regret[I] = agent.tmp_regret[I].copy() @@ -171,11 +172,32 @@ def rts( if no_info_set or offline_strategy[info_set] == {}: offline_strategy[info_set] = {a: 0 for a in strategy.keys()} for action, probability in strategy.items(): - try: - offline_strategy[info_set][action] += probability - except: - import ipdb; - ipdb.set_trace() + offline_strategy[info_set][action] += probability agent.reset_new_regret() return agent, offline_strategy + + +if __name__ == "__main__": + # We can set public cards or not + public_cards = [Card("ace", "diamonds"), Card("king", "clubs"), + Card("jack", "spades"), Card("10", "hearts"), + Card("10", "spades")] + # Action sequence must be in old form (one list, includes skips) + action_sequence = ["raise", "raise", "raise", "call", "call", + "raise", "raise", "raise", "call", "call", + "raise", "raise", "raise", "call", "call", "call"] + agent_output, offline_strategy = rts( + 'test_strategy3/unnormalized_output/offline_strategy_1500.gz', + 'test_strategy3/strategy.gz', public_cards, action_sequence, + 1400, 1, 1, 3, 1, 1, 20 + ) + save_path = "test_strategy3/unnormalized_output/" + last_regret = { + info_set: dict(strategy) + for info_set, strategy in agent_output.regret.items() + } + joblib.dump(offline_strategy, save_path + 'rts_output.gz', compress="gzip") + joblib.dump(last_regret, save_path + 'last_regret.gz', compress="gzip") + import ipdb; + ipdb.set_trace() diff --git a/research/stat_tests/bot_test.py b/research/stat_tests/agent_test.py similarity index 81% rename from research/stat_tests/bot_test.py rename to research/stat_tests/agent_test.py index 678470e4..6569ac6e 100644 --- a/research/stat_tests/bot_test.py +++ b/research/stat_tests/agent_test.py @@ -3,6 +3,7 @@ import joblib import collections +import click from tqdm import trange import yaml import datetime @@ -12,6 +13,7 @@ from pluribus.games.short_deck.state import ShortDeckPokerState, new_game from pluribus.poker.card import Card + def _calculate_strategy( state: ShortDeckPokerState, I: str, @@ -19,7 +21,9 @@ def _calculate_strategy( count=None, total_count=None ) -> str: - sigma = collections.defaultdict(lambda: collections.defaultdict(lambda: 1 / 3)) + sigma = collections.defaultdict( + lambda: collections.defaultdict(lambda: 1 / 3) + ) try: # If strategy is empty, go to other block sigma[I] = strategy[I].copy() @@ -59,7 +63,7 @@ def agent_test( real_time_est: bool = False, action_sequence: List[str] = None, public_cards: List[Card] = [], - n_outter_iters: int = 30, + n_outer_iters: int = 30, n_inner_iters: int = 100, n_players: int = 3, hero_count=None, @@ -84,22 +88,27 @@ def agent_test( opponent_strategy, action_sequence ) - # TODO: Right now, this can only be used for loading states if the two strategies - # are averaged. Even averaging strategies is risky. Loading a game state - # should be used with caution. It will work only if the probability of reach - # is identical across strategies. Use the average strategy. + # TODO: Right now, this can only be used for loading states if the two + # strategies are averaged. Even averaging strategies is risky. Loading a + # game state should be used with caution. It will work only if the + # probability of reach is identical across strategies. Use the average + # strategy. info_set_lut = {} EVs = np.array([]) - for _ in trange(1, n_outter_iters): + for _ in trange(1, n_outer_iters): EV = np.array([]) # Expected value for player 0 (hero) for t in trange(1, n_inner_iters + 1, desc="train iter"): for p_i in range(n_players): if real_time_est: - # Deal hole cards based on bayesian updating of hole card probs + # Deal hole cards based on bayesian updating of hole card + # probabilities state: ShortDeckPokerState = current_game_state.deal_bayes() else: - state: ShortDeckPokerState = new_game(n_players, info_set_lut) + state: ShortDeckPokerState = new_game( + n_players, + info_set_lut + ) info_set_lut = state.info_set_lut while True: player_not_in_hand = not state.players[p_i].is_active @@ -107,12 +116,13 @@ def agent_test( EV = np.append(EV, state.payout[p_i]) break if state.player_i == p_i: - random_action, hero_count, hero_total_count = _calculate_strategy( - state, - state.info_set, - hero_strategy, - count=hero_count, - total_count=hero_total_count + random_action, hero_count, hero_total_count = \ + _calculate_strategy( + state, + state.info_set, + hero_strategy, + count=hero_count, + total_count=hero_total_count ) else: random_action, oc, otc = _calculate_strategy( @@ -122,8 +132,8 @@ def agent_test( ) state = state.apply_action(random_action) EVs = np.append(EVs, EV.mean()) - t_stat = (EVs.mean() - 0) / (EVs.std() / np.sqrt(n_outter_iters)) - p_val = stats.t.sf(np.abs(t_stat), n_outter_iters - 1) + t_stat = (EVs.mean() - 0) / (EVs.std() / np.sqrt(n_outer_iters)) + p_val = stats.t.sf(np.abs(t_stat), n_outer_iters - 1) results_dict = { 'Expected Value': float(EVs.mean()), 'T Statistic': float(t_stat), @@ -146,7 +156,7 @@ def agent_test( public_cards=[], action_sequence=None, n_inner_iters=25, - n_outter_iters=75, + n_outer_iters=75, hero_count=0, hero_total_count=0 ) diff --git a/research/stat_tests/entry_rts_ab.py b/research/stat_tests/entry_rts_ab.py deleted file mode 100644 index df29136e..00000000 --- a/research/stat_tests/entry_rts_ab.py +++ /dev/null @@ -1,67 +0,0 @@ -import numpy as np -import json -import joblib - -from RT_cfr import rts -from bot_test import agent_test -from pluribus.poker.deck import Deck - - -check = joblib.load('test_strategy2/unnormalized_output/offline_strategy_1500.gz') -histories = np.random.choice(list(check.keys()), 2) -action_sequences = [] -public_cards_lst = [] -community_card_dict = { - "pre_flop": 0, - "flop": 3, - "turn": 4, - "river": 5, -} -ranks = list(range(10, 14 + 1)) -deck = Deck(include_ranks=ranks) -for history in histories: - history_dict = json.loads(history) - history_lst = history_dict['history'] - action_sequence = [] - betting_rounds = [] - for x in history_lst: - action_sequence += list(x.values())[0] - betting_rounds += list(x.keys()) - action_sequences.append(action_sequence) - final_betting_round = list(betting_rounds)[-1] - n_cards = community_card_dict[final_betting_round] - cards_in_deck = deck._cards_in_deck - public_cards = np.random.choice(cards_in_deck, n_cards) - public_cards_lst.append(list(public_cards)) - -for i in range(0, len(action_sequences)): -# try: - public_cards = public_cards_lst[i].copy() - action_sequence = action_sequences[i].copy() - agent_output, offline_strategy = rts( - 'test_strategy2/unnormalized_output/offline_strategy_1500.gz', - 'test_strategy2/strategy_1500.gz', public_cards, action_sequence, - 1500, 400, 400, 3, 400, 400, 20 - ) - save_path = "test_strategy2/unnormalized_output/" - last_regret = {info_set: dict(strategy) for info_set, strategy in agent_output.regret.items()} - joblib.dump(offline_strategy, save_path + f'rts_output{i+35}.gz', compress="gzip") - joblib.dump(last_regret, save_path + f'last_regret{i+35}.gz', compress="gzip") - - public_cards = public_cards_lst[i].copy() - action_sequence = action_sequences[i].copy() - strat_path = "test_strategy2/unnormalized_output/" - agent_test( - hero_strategy_path=strat_path + f"rts_output{i+35}.gz", - opponent_strategy_path=strat_path + "offline_strategy_1500.gz", - real_time_est=True, - public_cards=public_cards, - action_sequence=action_sequence, - n_inner_iters=25, - n_outter_iters=250, - hero_count=0, - hero_total_count=0, - ) -# except: -# print(f"ERROR on {action_sequences[i]}, {public_cards_lst[i]}") -# pass diff --git a/research/stat_tests/rts_ab_test.py b/research/stat_tests/rts_ab_test.py new file mode 100644 index 00000000..047d571d --- /dev/null +++ b/research/stat_tests/rts_ab_test.py @@ -0,0 +1,112 @@ +import numpy as np +import json +import joblib +import sys +from typing import List + +import click + +from agent_test import agent_test +from pluribus.poker.deck import Deck +sys.path.append('research/rts') +from RT_cfr import rts + + +@click.command() +@click.option("--offline_strategy_path", help=".") +@click.option("--last_regret_path", help=".") +@click.option("--n_iterations", default=1500, help=".") +@click.option("--lcfr_threshold", default=400, help=".") +@click.option("--discount_interval", default=400, help=".") +@click.option("--n_players", default=3, help=".") +@click.option("--update_interval", default=400, help=".") +@click.option("--update_threshold", default=400, help=".") +@click.option("--dump_int", default=20, help=".") +@click.option("--save_dir", help=".") +@click.option("--n_inner_iters", default=25, help=".") +@click.option("--n_outer_iters", default=150, help=".") +def rts_ab_test( + offline_strategy_path: str, + last_regret_path: str, + n_iterations: int, + lcfr_threshold: int, + discount_interval: int, + n_players: int, + update_interval: int, + update_threshold: int, + dump_int: int, + save_dir: str, + n_inner_iters: int, + n_outer_iters: int, + ranks: List[int] = list(range(10, 14 + 1)), +): + check = joblib.load(offline_strategy_path) + histories = np.random.choice(list(check.keys()), 2) + action_sequences = [] + public_cards_lst = [] + community_card_dict = { + "pre_flop": 0, + "flop": 3, + "turn": 4, + "river": 5, + } + deck = Deck(include_ranks=ranks) + for history in histories: + history_dict = json.loads(history) + history_lst = history_dict['history'] + action_sequence = [] + betting_rounds = [] + for x in history_lst: + action_sequence += list(x.values())[0] + betting_rounds += list(x.keys()) + action_sequences.append(action_sequence) + if action_sequences: + final_betting_round = list(betting_rounds)[-1] + else: + final_betting_round = "pre_flop" + n_cards = community_card_dict[final_betting_round] + cards_in_deck = deck._cards_in_deck + public_cards = list( + np.random.choice(cards_in_deck, n_cards) + ) + public_cards_lst.append(public_cards) + + for i in range(0, len(action_sequences)): + public_cards = public_cards_lst[i].copy() + action_sequence = action_sequences[i].copy() + agent_output, offline_strategy = rts( + offline_strategy_path, + last_regret_path, + public_cards, + action_sequence, + n_iterations=n_iterations, + lcfr_threshold=lcfr_threshold, + discount_interval=discount_interval, + n_players=n_players, + update_interval=update_interval, + update_threshold=update_threshold, + dump_int=dump_int + ) + last_regret = { + info_set: dict(strategy) + for info_set, strategy in agent_output.regret.items() + } + joblib.dump(offline_strategy, save_dir + f'rts_output{i}.gz', compress="gzip") + joblib.dump(last_regret, save_dir + f'last_regret{i}.gz', compress="gzip") + + public_cards = public_cards_lst[i].copy() + action_sequence = action_sequences[i].copy() + agent_test( + hero_strategy_path=save_dir + f"rts_output{i}.gz", + opponent_strategy_path=offline_strategy_path, + real_time_est=True, + public_cards=public_cards, + action_sequence=action_sequence, + n_inner_iters=n_inner_iters, + n_outer_iters=n_outer_iters, + hero_count=0, + hero_total_count=0, + ) + +if __name__ == "__main__": + rts_ab_test() diff --git a/test/functional/test_short_deck.py b/test/functional/test_short_deck.py index 06c679b5..276082ba 100644 --- a/test/functional/test_short_deck.py +++ b/test/functional/test_short_deck.py @@ -396,6 +396,7 @@ def test_public_cards( histories = np.random.choice(list(strategy.keys()), n) action_sequences = [] public_cards_lst = [] + final_betting_round_lst: List[str] = [] community_card_dict = { "pre_flop": 0, "flop": 3, @@ -412,12 +413,17 @@ def test_public_cards( for x in history_lst: action_sequence += list(x.values())[0] betting_rounds += list(x.keys()) + if not action_sequence: + continue action_sequences.append(action_sequence) final_betting_round = list(betting_rounds)[-1] + final_betting_round_lst.append(final_betting_round) n_cards = community_card_dict[final_betting_round] cards_in_deck = deck._cards_in_deck - public_cards = np.random.choice(cards_in_deck, n_cards, replace=False) - public_cards_lst.append(list(public_cards)) + public_cards = list( + np.random.choice(cards_in_deck, n_cards, replace=False) + ) + public_cards_lst.append(public_cards) # TODO: Not sure how to quiet mypy here for typing complaint.. info_set_lut: InfoSetLookupTable = { @@ -428,7 +434,8 @@ def test_public_cards( } for i in range(0, len(action_sequences)): public_cards = public_cards_lst[i].copy() - if not public_cards: + final_betting_round = final_betting_round_lst[i] + if not public_cards and final_betting_round == "pre_flop": continue action_sequence = action_sequences[i].copy() state: ShortDeckPokerState = new_game(