diff --git a/python/ray/rllib/optimizers/apex_optimizer.py b/python/ray/rllib/optimizers/apex_optimizer.py
index ded738f622fc..e113213b4940 100644
--- a/python/ray/rllib/optimizers/apex_optimizer.py
+++ b/python/ray/rllib/optimizers/apex_optimizer.py
@@ -7,12 +7,12 @@
 from __future__ import print_function
 
 import os
-import queue
 import random
 import time
 import threading
 
 import numpy as np
+from six.moves import queue
 
 import ray
 from ray.rllib.optimizers.policy_optimizer import PolicyOptimizer
@@ -22,6 +22,7 @@
 from ray.rllib.utils.timer import TimerStat
 from ray.rllib.utils.window_stat import WindowStat
 
+
 SAMPLE_QUEUE_DEPTH = 2
 REPLAY_QUEUE_DEPTH = 4
 LEARNER_QUEUE_MAX_SIZE = 16