diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index 7ac3cec3af..ea348ae126 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -8,6 +8,7 @@
 
 - Add `check_test_point` method to `pm.Model`
 - Add `Ordered` Transformation and `OrderedLogistic` distribution
+- Better warning message for `Mass matrix contains zeros on the diagonal. Some derivatives might always be zero`
 
 ### Fixes
 
diff --git a/pymc3/step_methods/hmc/base_hmc.py b/pymc3/step_methods/hmc/base_hmc.py
index d18e28cd0e..68d0627e8c 100644
--- a/pymc3/step_methods/hmc/base_hmc.py
+++ b/pymc3/step_methods/hmc/base_hmc.py
@@ -112,7 +112,7 @@ def astep(self, q0):
         start = self.integrator.compute_state(q0, p0)
 
         if not np.isfinite(start.energy):
-            self.potential.raise_ok()
+            self.potential.raise_ok(self._logp_dlogp_func._ordering.vmap)
             raise ValueError('Bad initial energy: %s. The model '
                              'might be misspecified.' % start.energy)
 
diff --git a/pymc3/step_methods/hmc/quadpotential.py b/pymc3/step_methods/hmc/quadpotential.py
index 1eaf165014..33a1d73f9f 100644
--- a/pymc3/step_methods/hmc/quadpotential.py
+++ b/pymc3/step_methods/hmc/quadpotential.py
@@ -185,15 +185,34 @@ def update(self, sample, grad, tune):
 
         self._n_samples += 1
 
-    def raise_ok(self):
+    def raise_ok(self, vmap):
         if np.any(self._stds == 0):
-            raise ValueError('Mass matrix contains zeros on the diagonal. '
-                             'Some derivatives might always be zero.')
-        if np.any(self._stds < 0):
-            raise ValueError('Mass matrix contains negative values on the '
-                             'diagonal.')
+            name_slc = []
+            tmp_hold = list(range(self._stds.size))
+            for vmap_ in vmap:
+                slclen = len(tmp_hold[vmap_.slc])
+                for i in range(slclen):
+                    name_slc.append((vmap_.var, i))
+            index = np.where(self._stds == 0)[0]
+            errmsg = ['Mass matrix contains zeros on the diagonal. ']
+            for ii in index:
+                errmsg.append('The derivative of RV `{}`.ravel()[{}]'
+                              ' is zero.'.format(*name_slc[ii]))
+            raise ValueError('\n'.join(errmsg))
+
         if np.any(~np.isfinite(self._stds)):
-            raise ValueError('Mass matrix contains non-finite values.')
+            name_slc = []
+            tmp_hold = list(range(self._stds.size))
+            for vmap_ in vmap:
+                slclen = len(tmp_hold[vmap_.slc])
+                for i in range(slclen):
+                    name_slc.append((vmap_.var, i))
+            index = np.where(~np.isfinite(self._stds))[0]
+            errmsg = ['Mass matrix contains non-finite values on the diagonal. ']
+            for ii in index:
+                errmsg.append('The derivative of RV `{}`.ravel()[{}]'
+                              ' is non-finite.'.format(*name_slc[ii]))
+            raise ValueError('\n'.join(errmsg))
 
 
 class QuadPotentialDiagAdaptGrad(QuadPotentialDiagAdapt):