Aimbot-PPO/Aimbot-PPO-Python/PPO.py

import tensorflow as tf
from tensorflow.python.ops.numpy_ops import ndarray
import tensorflow_probability as tfp
import numpy as np
import time
import math

import datetime
from PPOConfig import PPOConfig

from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras import optimizers

EPS = 1e-10


class PPO(object):
    def __init__(
        self,
        stateSize: int,
        disActShape: list,
        conActSize: int,
        conActRange: float,
        PPOConfig: PPOConfig,
    ):
        """initialize PPO

        Args:
            stateSize (int): enviroment state size
            disActShape (numpy): discrete Action shape.
                                just like [3,2],means 2 type of dis actions,each act include 3 and 2 types
                                if no discrete action output then use [0].
            conActSize (int): continuous Action Size. if no continuous action output then use 0.
            conActRange (float): continuous action range. -conActRange to +conActRange
            PPOConfig (PPOConfig): PPO configuration
        """
        # check use dis action or not.
        if disActShape == [0]:
            # non dis action output
            self.disActSize = 0
            self.disOutputSize = 0
        else:
            # make sure disActShape greater than 1
            try:
                if np.any(np.array(disActShape) <= 1):
                    raise ValueError(
                        "disActShape error,disActShape should greater than 1 but get", disActShape
                    )
            except ValueError:
                raise
            self.disActSize = len(disActShape)
            self.disOutputSize = sum(disActShape)

        self.stateSize = stateSize
        self.disActShape = disActShape
        self.conActSize = conActSize
        self.conActRange = conActRange
        self.muSigSize = 2
        self.conOutputSize = conActSize * self.muSigSize

        # config
        self.NNShape = PPOConfig.NNShape
        self.criticLR = PPOConfig.criticLR
        self.actorLR = PPOConfig.actorLR
        self.gamma = PPOConfig.gamma
        self.lmbda = PPOConfig.lmbda
        self.clipRange = PPOConfig.clipRange
        self.entropyWeight = PPOConfig.entropyWeight
        self.trainEpochs = PPOConfig.trainEpochs
        self.saveDir = PPOConfig.saveDir
        self.loadModelDir = PPOConfig.loadModelDir
        print("---------thisPPO Params---------")
        print("self.stateSize = ", self.stateSize)
        print("self.disActShape = ", self.disActShape)
        print("self.disActSize", self.disActSize)
        print("self.disOutputSize", self.disOutputSize)
        print("self.conActSize = ", self.conActSize)
        print("self.conActRange = ", self.conActRange)
        print("self.conOutputSize = ", self.conOutputSize)

        # config
        print("---------thisPPO config---------")
        print("self.NNShape = ", self.NNShape)
        print("self.criticLR = ", self.criticLR)
        print("self.actorLR = ", self.actorLR)
        print("self.gamma = ", self.gamma)
        print("self.lmbda = ", self.lmbda)
        print("self.clipRange = ", self.clipRange)
        print("self.entropyWeight = ", self.entropyWeight)
        print("self.trainEpochs = ", self.trainEpochs)
        print("self.saveDir = ", self.saveDir)
        print("self.loadModelDir = ", self.loadModelDir)

        # load NN or not
        if self.loadModelDir is None:
            # critc NN
            self.critic = self.buildCriticNet(self.stateSize, 1, compileModel=True)
            # actor NN
            self.actor = self.buildActorNet(self.stateSize, compileModel=True)
            print("---------Actor Model Create Success---------")
            self.actor.summary()
            print("---------Critic Model Create Success---------")
            self.critic.summary()
        else:
            # critc NN
            self.critic = self.buildCriticNet(self.stateSize, 1, compileModel=True)
            # actor NN
            self.actor = self.buildActorNet(self.stateSize, compileModel=True)
            # load weight to Critic&Actor NN
            self.loadWeightToModels(self.loadModelDir)
            print("---------Actor Model Load Success---------")
            self.actor.summary()
            print("---------Critic Model Load Success---------")
            self.critic.summary()

    # Build Net
    def buildActorNet(self, inputSize: int, compileModel: bool):
        """build Actor Nueral Net and compile.Output:[disAct1,disAct2,disAct3,mu,sigma]

        Args:
            inputSize (int): InputLayer Nueral size.
            compileModel (bool): compile Model or not.

        Returns:
            keras.Model: return Actor NN
        """
        # -----------Input Layers-----------
        stateInput = layers.Input(shape=(inputSize,), name="stateInput")

        # -------Intermediate layers--------
        interLayers = []
        interLayersIndex = 0
        for neuralUnit in self.NNShape:
            thisLayerName = "dense" + str(interLayersIndex)
            if interLayersIndex == 0:
                interLayers.append(
                    layers.Dense(neuralUnit, activation="relu", name=thisLayerName)(stateInput)
                )
            else:
                interLayers.append(
                    layers.Dense(neuralUnit, activation="relu", name=thisLayerName)(interLayers[-1])
                )
            interLayersIndex += 1

        # ----------Output Layers-----------
        outputLayersList = []
        if self.disActSize != 0:
            # while NN have discrete action output.
            disActIndex = 0
            for thisDisActDepth in self.disActShape:
                thisDisActName = "disAct" + str(disActIndex)
                outputLayersList.append(
                    layers.Dense(thisDisActDepth, activation="softmax", name=thisDisActName)(
                        interLayers[-1]
                    )
                )
                disActIndex += 1
        if self.conActSize != 0:
            # while NN have continuous action output.
            mu = tf.multiply(
                layers.Dense(1, activation="tanh", name="muOut")(interLayers[-1]), self.conActRange
            )  # mu，既正态分布位置参数
            sigma = tf.add(
                layers.Dense(1, activation="softplus", name="sigmaOut")(interLayers[-1]), EPS
            )  # sigma，既正态分布尺度参数
            outputLayersList.append(mu)
            outputLayersList.append(sigma)
        totalOut = layers.concatenate(outputLayersList, name="totalOut")  # package

        # ----------Model Compile-----------
        model = keras.Model(inputs=stateInput, outputs=totalOut)
        if compileModel:  # Compile Model
            actorOPT = optimizers.Adam(learning_rate=self.actorLR)
            model.compile(optimizer=actorOPT, loss=self.aLoss())
        return model

    def buildCriticNet(self, inputSize: int, outputSize: int, compileModel: bool):
        """build Critic Nueral Net and compile.Output:[Q]

        Args:
            inputSize (int): input size
            outputSize (int): output size
            compileModel (bool): compile Model or not.

        Returns:
            keras.Model: return Critic NN
        """
        # -----------Input Layers-----------
        stateInput = keras.Input(shape=(inputSize,), name="stateInput")

        # -------Intermediate layers--------
        interLayers = []
        interLayersIndex = 0
        for neuralUnit in self.NNShape:
            thisLayerName = "dense" + str(interLayersIndex)
            if interLayersIndex == 0:
                interLayers.append(
                    layers.Dense(neuralUnit, activation="relu", name=thisLayerName)(stateInput)
                )
            else:
                interLayers.append(
                    layers.Dense(neuralUnit, activation="relu", name=thisLayerName)(interLayers[-1])
                )
            interLayersIndex += 1

        # ----------Output Layers-----------
        output = layers.Dense(outputSize, activation=None)(interLayers[-1])

        # ----------Model Compile-----------
        model = keras.Model(inputs=stateInput, outputs=output)
        if compileModel:
            criticOPT = optimizers.Adam(learning_rate=self.criticLR)
            model.compile(optimizer=criticOPT, loss=self.cLoss())
        return model

    # loss Function
    # critic loss
    def cLoss(self):
        """Critic Loss function"""

        def loss(y_true, y_pred):
            # y_true: discountedR
            # y_pred: critcV = model.predict(states)

            adv = y_true - y_pred  # TD error
            loss = tf.reduce_mean(tf.square(adv))
            return loss

        return loss

    # actor loss
    def aLoss(self):
        """Actor Loss function"""

        def getDiscreteALoss(nowProbs, oldProbs, disOneHotAct, actShape, advantage):
            """get Discrete Action Loss

            Args:
                nowProbs (tf.constant): (length,actionProbSize)
                oldProbs (tf.constant): (length,actionProbSize)
                advantage (tf.constant): (length,)

            Returns:
                tf.constant: (length,)
            """
            entropy = tf.negative(
                tf.reduce_mean(tf.math.multiply(nowProbs, tf.math.log(nowProbs + EPS)))
            )
            nowSingleProbs = tf.reduce_mean(tf.multiply(nowProbs, disOneHotAct), axis=1)
            nowSingleProbs = tf.multiply(nowSingleProbs, actShape)
            oldSingleProbs = tf.reduce_mean(tf.multiply(oldProbs, disOneHotAct), axis=1)
            oldSingleProbs = tf.multiply(oldSingleProbs, actShape)
            ratio = tf.math.divide(nowSingleProbs, oldSingleProbs + EPS)
            value = tf.math.multiply(ratio, advantage)
            clipRatio = tf.clip_by_value(ratio, 1.0 - self.clipRange, 1.0 + self.clipRange)
            clipValue = tf.math.multiply(clipRatio, advantage)
            loss = tf.math.negative(
                tf.reduce_mean(tf.math.minimum(value, clipValue))
                - tf.multiply(self.entropyWeight, entropy)
            )
            return loss

        def getContinuousALoss(musig, actions, oldProbs, advantage):
            """get Continuous Action Loss

            Args:
                musig (tf.constant): (length,2)
                actions (tf.constant): (length,)
                oldProbs (tf.constant): (length,)
                advantage (tf.constant): (length,)

            Returns:
                tf.constant: (length,)
            """
            mu = musig[:, 0]
            sigma = musig[:, 1]
            dist = tfp.distributions.Normal(mu, sigma)

            nowProbs = dist.prob(actions)
            entropy = tf.reduce_mean(dist.entropy())

            ratio = tf.math.divide(nowProbs, oldProbs + EPS)
            value = tf.math.multiply(ratio, advantage)
            clipRatio = tf.clip_by_value(ratio, 1.0 - self.clipRange, 1.0 + self.clipRange)
            clipValue = tf.math.multiply(clipRatio, advantage)
            loss = tf.negative(
                tf.reduce_mean(tf.math.minimum(value, clipValue))
                - tf.multiply(self.entropyWeight, entropy)
            )
            return loss

        def loss(y_true, y_pred):
            # y_true: [[disActProb..., conActProbs..., disOneHotActs..., conAct..., advantage]]
            # y_pred: [[disActProb..., mu, sigma...]]
            totalALoss = 0
            totalActionNum = 0
            advantage = tf.expand_dims(y_true[:, -1], axis=1)

            if self.disActSize != 0:
                # while NN have discrete action output.
                oldDisProbs = y_true[:, 0 : self.disOutputSize]
                nowDisProbs = y_pred[:, 0 : self.disOutputSize]  # [disAct1, disAct2, disAct3]
                disOneHotActs = y_true[
                    :,
                    self.disOutputSize
                    + self.conActSize : self.disOutputSize
                    + self.conActSize
                    + self.disOutputSize,
                ]
                lastDisActShape = 0
                for thisShape in self.disActShape:
                    thisNowDisProbs = nowDisProbs[:, lastDisActShape : lastDisActShape + thisShape]
                    thisOldDisProbs = oldDisProbs[:, lastDisActShape : lastDisActShape + thisShape]
                    thisDisOneHotActs = disOneHotActs[
                        :, lastDisActShape : lastDisActShape + thisShape
                    ]
                    discreteALoss = getDiscreteALoss(
                        thisNowDisProbs, thisOldDisProbs, thisDisOneHotActs, thisShape, advantage
                    )
                    lastDisActShape += thisShape
                    totalALoss += discreteALoss
                    totalActionNum += 1.0
            if self.conActSize != 0:
                # while NN have continuous action output.
                oldConProbs = y_true[:, self.disOutputSize : self.disOutputSize + self.conActSize]
                conActions = y_true[
                    :,
                    self.disOutputSize
                    + self.conActSize : self.disOutputSize
                    + self.conActSize
                    + self.conActSize,
                ]
                nowConMusigs = y_pred[:, self.disOutputSize :]  # [musig1,musig2]
                lastConAct = 0
                for conAct in range(self.conActSize):
                    thisNowConMusig = nowConMusigs[:, lastConAct : lastConAct + self.muSigSize]
                    thisOldConProb = oldConProbs[:, conAct : conAct + 1]
                    thisConAction = conActions[:, conAct]
                    continuousAloss = getContinuousALoss(
                        thisNowConMusig, thisConAction, thisOldConProb, advantage
                    )
                    totalALoss += continuousAloss
                    totalActionNum += 1.0
                    lastConAct += self.muSigSize
            # loss = tf.divide(totalALoss, totalActionNum)
            return totalALoss

        return loss

    # get Actions&values
    def chooseAction(self, state: ndarray):
        """Agent choose action to take

        Args:
            state (ndarray): enviroment state

        Returns:
            np.array:
                actions,
                    actions list,1dims like [0,1,1.5]
                predictResult,
                    actor NN predict Result output
        """
        # let actor choose action,use the normal distribution
        # state = np.expand_dims(state,0)

        # check state dimension is [stateNum,statesize]
        if state.ndim != 2:
            stateNum = int(len(state) / self.stateSize)
            state = state.reshape([stateNum, self.stateSize])
        predictResult = self.actor(state)  # get predict result [[disAct1, disAct2, disAct3, musig]]
        # print("predictResult",predictResult)
        # predictResult = predictResult.numpy()
        actions = []
        if self.disActSize != 0:
            # while NN have discrete action output.
            lastDisActShape = 0
            for shape in self.disActShape:
                thisDisActProbs = predictResult[:, lastDisActShape : lastDisActShape + shape]
                dist = tfp.distributions.Categorical(probs=thisDisActProbs, dtype=tf.float32)
                action = int(dist.sample().numpy()[0])
                # action = np.argmax(thisDisActProbs)
                actions.append(action)
                lastDisActShape += shape
        if self.conActSize != 0:
            # while NN have continuous action output.
            lastConAct = 0
            for actIndex in range(self.conActSize):
                thisMu = predictResult[:, self.disOutputSize + lastConAct]
                thisSig = predictResult[:, self.disOutputSize + lastConAct + 1]
                if math.isnan(thisMu) or math.isnan(thisSig):
                    # check mu or sigma is nan
                    print("chooseAction:mu or sigma is nan")
                    print(predictResult)
                thisDist = np.random.normal(loc=thisMu, scale=thisSig)
                actions.append(np.clip(thisDist, -self.conActRange, self.conActRange)[0])
                lastConAct += 2
        return actions, predictResult

    def trainCritcActor(
        self,
        states: ndarray,
        oldActorResult: ndarray,
        actions: ndarray,
        rewards: ndarray,
        dones: ndarray,
        nextState: ndarray,
        epochs: int = None,
    ):
        """train critic&actor use PPO ways

        Args:
            states (ndarray): states
            oldActorResult (ndarray): actor predict result
            actions (ndarray): predicted actions include both discrete actions and continuous actions
            rewards (ndarray): rewards from enviroment
            dones (ndarray): dones from enviroment
            nextState (ndarray): next state from enviroment
            epochs (int, optional): train epochs,default to ppoConfig. Defaults to None.

        Returns:
            tf.constant: criticLoss, actorLoss
        """

        if epochs == None:
            epochs = self.trainEpochs
        criticValues = self.getCriticV(state=states)
        discountedR = self.discountReward(nextState, criticValues, dones, rewards)
        advantage = self.getGAE(discountedR, criticValues)

        criticLoss = self.trainCritic(states, discountedR, epochs)
        actorLoss = self.trainActor(states, oldActorResult, actions, advantage, epochs)
        # print("A_Loss:", actorLoss, "C_Loss:", criticLoss)
        return criticLoss, actorLoss

    def trainCritic(self, states: ndarray, discountedR: ndarray, epochs: int = None):
        """critic NN trainning function

        Args:
            states (ndarray): states
            discountedR (ndarray): discounted rewards
            epochs (int, optional): train epochs,default to ppoConfig. Defaults to None.

        Returns:
            tf.constant: all critic losses
        """
        if epochs == None:
            epochs = self.trainEpochs
        his = self.critic.fit(x=states, y=discountedR, epochs=epochs, verbose=0)
        return his.history["loss"]

    def trainActor(
        self,
        states: ndarray,
        oldActorResult: ndarray,
        actions: ndarray,
        advantage: ndarray,
        epochs: int = None,
    ):
        """actor NN trainning function

        Args:
            states (ndarray): states
            oldActorResult (ndarray): actor predict results
            actions (ndarray): acotor predict actions
            advantage (ndarray): GAE advantage
            epochs (int, optional): train epochs,default to ppoConfig. Defaults to None.

        Returns:
            tf.constant: all actor losses
        """
        # Trian Actor
        # states: Buffer States
        # actions: Buffer Actions
        # discountedR: Discounted Rewards
        # Epochs: just Epochs
        if epochs == None:
            epochs = self.trainEpochs
        actions = np.asarray(actions, dtype=np.float32)

        disActions = actions[:, 0 : self.disActSize]
        conActions = actions[:, self.disActSize :]
        oldDisProbs = oldActorResult[:, 0 : self.disOutputSize]  # [disAct1, disAct2, disAct3]
        oldConMusigs = oldActorResult[:, self.disOutputSize :]  # [musig1,musig2]
        if self.disActSize != 0:
            disOneHotActs = self.getOneHotActs(disActions)
            if self.conActSize != 0:
                # while NN have discrete6 & continuous actions output.
                oldPiProbs = self.conProb(oldConMusigs[:, 0], oldConMusigs[:, 1], conActions)
                # pack [oldDisProbs,oldPiProbs,conActions,advantage] as y_true
                y_true = np.hstack((oldDisProbs, oldPiProbs, disOneHotActs, conActions, advantage))
            else:
                # while NN have only discrete actions output.
                # pack [oldDisProbs,advantage] as y_true
                y_true = np.hstack((oldDisProbs, disOneHotActs, advantage))
        else:
            if self.conActSize != 0:
                # while NN have only continuous action output.
                oldPiProbs = self.conProb(oldConMusigs[:, 0], oldConMusigs[:, 1], conActions)
                # pack [oldPiProbs,conActions,advantage] as y_true
                y_true = np.hstack((oldPiProbs, conActions, advantage))
            else:
                print("trainActor:disActSize & conActSize error")
                time.sleep(999999)
        # assembly Actions history
        # train start
        if np.any(tf.math.is_nan(y_true)):
            print("y_true got nan")
            print("y_true", y_true)
        his = self.actor.fit(x=states, y=y_true, epochs=epochs, verbose=0)
        if np.any(tf.math.is_nan(his.history["loss"])):
            print("his.history['loss'] is nan!")
            print(his.history["loss"])
        return his.history["loss"]

    def saveWeights(self, score: float):
        """save now NN's Weight. Use "models.save_weights" method.
        Save as "tf" format "ckpt" file.

        Args:
            score (float): now score
        """
        actor_save_dir = (
            self.saveDir + datetime.datetime.now().strftime("%H%M%S") + "/actor/" + "actor.ckpt"
        )
        critic_save_dir = (
            self.saveDir + datetime.datetime.now().strftime("%H%M%S") + "/critic/" + "critic.ckpt"
        )
        self.actor.save_weights(actor_save_dir, save_format="tf")
        self.critic.save_weights(critic_save_dir, save_format="tf")
        # create an empty file named  as score to recored score
        score_dir = (
            self.saveDir + datetime.datetime.now().strftime("%H%M%S") + "/" + str(round(score))
        )
        scorefile = open(score_dir, "w")
        scorefile.close()
        print("PPO Model's Weights Saved")

    def loadWeightToModels(self, loadDir: str):
        """load NN Model. Use "models.load_weights()" method.
        Load "tf" format "ckpt" file.

        Args:
            loadDir (str): Model dir
        """
        actorDir = loadDir + "/actor/" + "actor.ckpt"
        criticDir = loadDir + "/critic/" + "critic.ckpt"
        self.actor.load_weights(actorDir)
        self.critic.load_weights(criticDir)

        print("++++++++++++++++++++++++++++++++++++")
        print("++++++++++++Model Loaded++++++++++++")
        print(loadDir)
        print("++++++++++++++++++++++++++++++++++++")

    def getCriticV(self, state: ndarray):
        """get Critic predict V value

        Args:
            state (ndarray): Env state

        Returns:
            tensor: retrun Critic predict result
        """
        # if state.ndim < 2:
        #    state = np.expand_dims(state,0)
        if state.ndim != 2:
            stateNum = int(len(state) / self.stateSize)
            state = state.reshape([stateNum, self.stateSize])
        return self.critic.predict(state)

    def discountReward(self, nextState: ndarray, values: ndarray, dones: ndarray, rewards: ndarray):
        """Discount future rewards

        Args:
            nextState (ndarray): next Env state
            values (ndarray): critic predict values
            dones (ndarray): dones from enviroment
            rewards (ndarray): reward list of this episode

        Returns:
            ndarray: discounted rewards list,same shape as rewards that input
        """
        """
        nextV = self.getCriticV(nextState)
        dones = 1 - dones
        discountedRewards = []
        for i in reversed(range(len(rewards))):
            nextV = rewards[i] + dones[i] * self.gamma * nextV
            discountedRewards.append(nextV)
        discountedRewards.reverse()  # reverse
        discountedRewards = np.squeeze(discountedRewards)
        discountedRewards = np.expand_dims(discountedRewards, axis=1)
        # discountedRewards = np.array(discountedRewards)[:, np.newaxis]
        return discountedRewards
        """
        """
        nextV = self.getCriticV(nextState)
        discountedRewards = []
        for r in rewards[::-1]:
            nextV = r + self.gamma * nextV
            discountedRewards.append(nextV)
        discountedRewards.reverse()  # reverse
        discountedRewards = np.squeeze(discountedRewards)
        discountedRewards = np.expand_dims(discountedRewards, axis=1)
        # discountedRewards = np.array(discountedRewards)[:, np.newaxis]
        print(discountedRewards)
        return discountedRewards
        """
        g = 0
        discountedRewards = []
        lastValue = self.getCriticV(nextState)
        values = np.append(values, lastValue, axis=0)
        dones = 1 - dones
        for i in reversed(range(len(rewards))):
            delta = rewards[i] + self.gamma * values[i + 1] * dones[i] - values[i]
            g = delta + self.gamma * self.lmbda * dones[i] * g
            discountedRewards.append(g + values[i])
        discountedRewards.reverse()
        return np.asarray(discountedRewards)

    def getGAE(self, discountedRewards: ndarray, values: ndarray):
        """compute GAE adcantage

        Args:
            discountedRewards (ndarray): discounted rewards
            values (ndarray): critic predict values

        Returns:
            ndarray: GAE advantage
        """
        advantage = discountedRewards - values
        advantage = (advantage - np.mean(advantage)) / (np.std(advantage) + EPS)
        return advantage

    def conProb(self, mu: ndarray, sig: ndarray, x: ndarray):
        """calculate probability when x in Normal distribution(mu,sigma)

        Args:
            mu (ndarray): mu
            sig (ndarray): sigma
            x (ndarray): x

        Returns:
            ndarray: probability
        """
        # 获取在正态分布mu,sig下当取x值时的概率
        # return shape : (length,1)
        mu = np.reshape(mu, (np.size(mu),))
        sig = np.reshape(sig, (np.size(sig),))
        x = np.reshape(x, (np.size(x),))

        dist = tfp.distributions.Normal(mu, sig)
        prob = dist.prob(x)

        prob = np.reshape(prob, (np.size(x), 1))
        # dist = 1./(tf.sqrt(2.*np.pi)*sig)
        # prob = dist*tf.exp(-tf.square(x-mu)/(2.*tf.square(sig)))
        return prob

    def getOneHotActs(self, disActions):
        """one hot action encoder

        Args:
            disActions (ndarray): discrete actions

        Returns:
            ndarray: one hot actions
        """
        actIndex = 0
        for thisShape in self.disActShape:
            thisActs = disActions[:, actIndex]
            thisOneHotAct = tf.squeeze(tf.one_hot(thisActs, thisShape)).numpy()
            if actIndex == 0:
                oneHotActs = thisOneHotAct
            else:
                oneHotActs = np.append(oneHotActs, thisOneHotAct, axis=1)
            actIndex += 1
        return oneHotActs

    def getAverageEntropy(self, probs: ndarray):
        """get average dis&con ACT Entropys

        Args:
            probs (ndarray): actor NN predict result

        Returns:
            float: average total entropy
            list: discrete entropys
            list: continuous entropys
        """
        discreteEntropys = []
        continuousEntropys = []
        if self.disActSize != 0:
            disProbs = probs[:, 0 : self.disOutputSize]
            lastDisActIndex = 0
            for actShape in self.disActShape:
                thisDisProbs = disProbs[:, lastDisActIndex : lastDisActIndex + actShape]
                lastDisActIndex += actShape
                discreteEntropys.append(
                    tf.negative(
                        tf.reduce_mean(
                            tf.math.multiply(thisDisProbs, tf.math.log(thisDisProbs + EPS))
                        )
                    )
                )
        if self.conActSize != 0:
            conProbs = probs[:, self.disOutputSize :]
            conActIndex = 0
            for i in range(self.conActSize):
                thisConProbs = conProbs[:, conActIndex : conActIndex + 2]
                conActIndex += 2
                continuousEntropys.append(tf.reduce_mean(thisConProbs[:, 1]))
        averageEntropy = np.mean([np.mean(discreteEntropys), np.mean(continuousEntropys)])
        return averageEntropy, discreteEntropys, continuousEntropys