Merge entropy branch

async/A3CAgent.lua

@@ -88,13 +88,17 @@ function A3CAgent:accumulateGradients(terminal, state)
     probability:add(TINY_EPSILON) -- could contain 0 -> log(0)= -inf -> theta = nans
 
     self.vTarget[1] = -0.5 * (R - V)
-    
+
     -- ∇θ logp(s) = 1/p(a) for chosen a, 0 otherwise
     self.policyTarget:zero()
-    local logProbability = torch.log(probability)
-    -- Add (negative of) gradient of entropy of policy to target to improve exploration (prevent convergence to suboptimal deterministic policy)
-    self.policyTarget[action] = -(R - V) / probability[action]  - self.beta * logProbability:sum()
+    -- f(s) ∇θ logp(s)
+    self.policyTarget[action] = -(R - V) / probability[action] -- Negative target for gradient descent
 
+    -- Calculate (negative of) gradient of entropy of policy (for gradient descent): -(-logp(s) - 1)
+    local gradEntropy = torch.log(probability) + 1
+    -- Add to target to improve exploration (prevent convergence to suboptimal deterministic policy)
+    self.policyTarget:add(self.beta, gradEntropy)
+
     self.policyNet_:backward(self.states[i], self.targets)
   end
 end