Fixed a bug: Pi output in tensorflow is incorrect (missing softmax la…

connect4/keras/Connect4NNet.py

@@ -0,0 +1,45 @@
+import sys
+
+sys.path.append('..')
+
+from tensorflow.keras.layers import Input, Dense, Reshape, Conv2D, \
+    BatchNormalization, Flatten, Dropout, Activation
+from tensorflow.keras.optimizers import Adam
+from tensorflow.keras.models import Model
+
+
+class Connect4NNet:
+    def __init__(self, game, args):
+        # game params
+        self.board_x, self.board_y = game.getBoardSize()
+        self.action_size = game.getActionSize()
+        self.args = args
+        self.model = self.build_model()
+
+    def build_model(self):
+        self.input_boards = Input(shape=(self.board_x, self.board_y))
+        x_image = Reshape((self.board_x, self.board_y, 1))(self.input_boards)
+
+        h_conv1 = Activation('relu')(BatchNormalization(axis=3)(
+            Conv2D(self.args.num_channels, 3, padding='same', use_bias=False)(
+                x_image)))  # batch_size  x board_x x board_y x num_channels
+        h_conv2 = Activation('relu')(BatchNormalization(axis=3)(
+            Conv2D(self.args.num_channels, 3, padding='same', use_bias=False)(
+                h_conv1)))  # batch_size  x board_x x board_y x num_channels
+        h_conv3 = Activation('relu')(BatchNormalization(axis=3)(
+            Conv2D(self.args.num_channels, 3, padding='valid', use_bias=False)(
+                h_conv2)))  # batch_size  x (board_x-2) x (board_y-2) x num_channels
+        h_conv4 = Activation('relu')(BatchNormalization(axis=3)(
+            Conv2D(self.args.num_channels, 3, padding='valid', use_bias=False)(
+                h_conv3)))  # batch_size  x (board_x-4) x (board_y-4) x num_channels
+        h_conv4_flat = Flatten()(h_conv4)
+        s_fc1 = Dropout(self.args.dropout)(Activation('relu')(
+            BatchNormalization(axis=1)(Dense(1024, use_bias=False)(h_conv4_flat))))  # batch_size x 1024
+        s_fc2 = Dropout(self.args.dropout)(
+            Activation('relu')(BatchNormalization(axis=1)(Dense(512, use_bias=False)(s_fc1))))  # batch_size x 512
+        self.pi = Dense(self.action_size, activation='softmax', name='pi')(s_fc2)
+        self.v = Dense(1, activation='tanh', name='v')(s_fc2)
+        model = Model(inputs=self.input_boards, outputs=[self.pi, self.v])
+        model.compile(optimizer=Adam(self.args.lr),
+                      loss={'pi': 'categorical_crossentropy', 'v': 'mean_squared_error'})
+        return model

connect4/keras/NNet.py

@@ -0,0 +1,70 @@
+import os
+import sys
+import time
+
+import numpy as np
+from tqdm import tqdm
+
+sys.path.append('../../')
+from utils import *
+from NeuralNet import NeuralNet
+
+import tensorflow as tf
+from .Connect4NNet import Connect4NNet as onnet
+
+args = dotdict({
+    'lr': 0.001,
+    'dropout': 0.3,
+    'epochs': 10,
+    'batch_size': 64,
+    'num_channels': 512,
+})
+
+
+class NNetWrapper(NeuralNet):
+    def __init__(self, game):
+        self.nnet = onnet(game, args)
+        self.board_x, self.board_y = game.getBoardSize()
+        self.action_size = game.getActionSize()
+
+        self.saver = None
+
+    def train(self, examples):
+        """
+        examples: list of examples, each example is of form (board, pi, v)
+        """
+        input_boards, target_pis, target_vs = list(zip(*examples))
+        input_boards = np.asarray(input_boards)
+        target_pis = np.asarray(target_pis)
+        target_vs = np.asarray(target_vs)
+        self.nnet.model.fit(x=input_boards, y={'pi': target_pis, 'v': target_vs},
+                            batch_size=args.batch_size, epochs=args.epochs)
+
+    def predict(self, board):
+        """
+        board: np array with board
+        """
+
+        # preparing input
+        board = board[np.newaxis, :, :]
+
+        # run
+        pi, v = self.nnet.model.predict(board)
+
+        return pi[0], v[0]
+
+    def save_checkpoint(self, folder='checkpoint', filename='checkpoint.pth.tar'):
+        filepath = os.path.join(folder, filename)
+        if not os.path.exists(folder):
+            print("Checkpoint Directory does not exist! Making directory {}".format(folder))
+            os.mkdir(folder)
+        else:
+            print("Checkpoint Directory exists! ")
+        self.nnet.model.save_weights(filepath)
+
+    def load_checkpoint(self, folder='checkpoint', filename='checkpoint.pth.tar'):
+        # https://github.com/pytorch/examples/blob/master/imagenet/main.py#L98
+        filepath = os.path.join(folder, filename)
+        # if not os.path.exists(filepath):
+        #     raise ("No model in path {}".format(filepath))
+        self.nnet.model.load_weights(filepath)

othello/tensorflow/OthelloNNet.py

@@ -33,8 +33,7 @@ def __init__(self, game, args):
             h_conv4_flat = tf.reshape(h_conv4, [-1, args.num_channels*(self.board_x-4)*(self.board_y-4)])
             s_fc1 = Dropout(Relu(BatchNormalization(Dense(h_conv4_flat, 1024, use_bias=False), axis=1, training=self.isTraining)), rate=self.dropout) # batch_size x 1024
             s_fc2 = Dropout(Relu(BatchNormalization(Dense(s_fc1, 512, use_bias=False), axis=1, training=self.isTraining)), rate=self.dropout)         # batch_size x 512
-            self.pi = Dense(s_fc2, self.action_size)                                                        # batch_size x self.action_size
-            self.prob = tf.nn.softmax(self.pi)
+            self.pi = tf.nn.softmax(Dense(s_fc2, self.action_size))                                                     # batch_size x self.action_size
             self.v = Tanh(Dense(s_fc2, 1))                                                               # batch_size x 1
 
             self.calculate_loss()