QLearnerSER.py

import random
import numpy as np
from scipy.optimize import minimize
from QLearnerESR import scalarize


class QLearnerSER:

    def __init__(self, agent_id, alpha, gamma, epsilon, num_states, num_actions, num_objectives, opt=False,
                 rand_prob=False):
        self.agent_id = agent_id
        self.alpha = alpha
        self.gamma = gamma
        self.epsilon = epsilon
        self.num_states = num_states
        self.num_actions = num_actions
        self.num_objectives = num_objectives
        # optimistic initialization of Q-table
        if opt:
            self.q_table = np.ones((num_states, num_actions, num_objectives)) * 20
        else:
            self.q_table = np.zeros((num_states, num_actions, num_objectives))
        self.current_state = -1
        self.rand_prob = rand_prob

    def update_q_table(self, prev_state, action, curr_state, reward):
        old_q = self.q_table[prev_state][action]
        next_q = self.q_table[curr_state, :, :]
        new_q = np.zeros(self.num_objectives)
        for o in range(self.num_objectives):
            new_q[o] = old_q[o] + self.alpha * (reward[o] + self.gamma * max(next_q[:, o]) - old_q[o])
            self.q_table[prev_state][action][o] = new_q[o]

    # random action selection
    def select_random_action(self):
        random_action = np.random.randint(self.num_actions)
        return random_action

    # epsilon greedy based on nonlinear optimiser mixed strategy search
    def select_action_mixed_nonlinear(self, state):
        self.current_state = state
        if random.uniform(0.0, 1.0) < self.epsilon:
            return self.select_random_action()
        else:
            return self.select_action_greedy_mixed_nonlinear(state)

    # greedy action selection based on nonlinear optimiser mixed strategy search
    def select_action_greedy_mixed_nonlinear(self, state):
        strategy = self.calc_mixed_strategy_nonlinear(state)
        if isinstance(strategy, int) or isinstance(strategy, np.int64):
            return strategy
        else:
            if np.sum(strategy) != 1:
                strategy = strategy / np.sum(strategy)
            return np.random.choice(range(self.num_actions), p=strategy)

    def calc_mixed_strategy_nonlinear(self, state):
        if self.rand_prob:
            s0 = np.random.random(self.num_actions)
            s0 /= np.sum(s0)
        else:
            s0 = np.full(self.num_actions, 1.0 / self.num_actions)  # initial guess set to equal prob over all actions

        b = (0.0, 1.0)
        bnds = (b,) * self.num_actions
        con1 = {'type': 'eq', 'fun': lambda x: np.sum(x) - 1}
        cons = ([con1])
        solution = minimize(self.objective, s0, bounds=bnds, constraints=cons)
        strategy = solution.x
        return strategy

    # this is the objective function to be minimised by the nonlinear optimiser
    # (therefore it returns the negative of SER)
    def objective(self, strategy):
        return - self.calc_ser_from_strategy(strategy)

    # Calculates the SER for a given strategy using the agent's own Q values
    def calc_ser_from_strategy(self, strategy):
        expected_vec = self.calc_expected_vec(self.current_state, strategy)
        ser = scalarize(self.agent_id, expected_vec)
        return ser

    # Calculates the expected payoff vector for a given strategy using the agent's own Q values
    def calc_expected_vec(self, state, strategy):
        expected_vec = np.zeros(self.num_objectives)
        for o in range(self.num_objectives):
            expected_vec[o] = np.dot(self.q_table[state, :, o], np.array(strategy))
        return expected_vec

    @staticmethod
    def select_recommended_action(state):
        return state