title: How to fix RuntimeError: deque mutated during iteration in Python
body: <p>I'm trying to implement market simulation for algorithmic trading and I've found this code on github <a href="https://github.com/DrAshBooth/PyLOB" rel="nofollow noreferrer">https://github.com/DrAshBooth/PyLOB</a>.
  The problem is when I'm running my code for small window, for example, 2 days, everything is fine and I get my expected results. But when I increase window to 20 days or more, I get "RuntimeError: deque mutated during iteration".
  I've checked my code but I've never found anything that could mutate deque during my runs.
  Below is the part of the code that produces the error:</p>
  
  <pre><code>    self.tape = deque(maxlen=None)
      .
      .
      .
      def avg_volume_traded(self):
         if self.tape != None and len(self.tape) &gt; 0:
            num = 0
            total_volume = 0
            for entry in self.tape:
               total_volume = entry['qty'] + total_volume
               num += 1
            if num != 0 and total_volume != None:
               return total_volume, num
         else:
            return None, None
  </code></pre>
  
  <p>This is the actual error message:</p>
  
  <pre><code>    Exception in thread Thread-10986:
      Traceback (most recent call last):
        File "/home/hamid/anaconda3/lib/python3.6/threading.py", line 916, in _bootstrap_inner
          self.run()
        File "/home/hamid/anaconda3/lib/python3.6/threading.py", line 864, in run
          self._target(*self._args, **self._kwargs)
        File "exchange_envirnment.py", line 60, in _doit
          self.func(self.args[0], self.args[1])
        File "/home/hamid/dukto/src2/src_new/traders/market_maker_trader.py", line 46, in trade
          self.type_three(lob_obj, reporter_obj)
        File "/home/hamid/dukto/src2/src_new/traders/market_maker_trader.py", line 285, in type_three
          max_volume = lob_obj.max_volume_traded()
        File "/home/hamid/dukto/src2/src_new/PyLOB/orderbook.py", line 395, in max_volume_traded
          for entry in self.tape:
      RuntimeError: deque mutated during iteration
  
  </code></pre>
  
  <p>This is the main part that use threading in two parts(class Periodic and day_period):</p>
  
  <pre><code>class Periodic(object):
      def __init__(self, object, compression_factor, args=[], kwargs={}):
          self.compression_factor = compression_factor
          self.object = object
          self.func = object.trade
          self.args = args
          self.kwargs = kwargs
          self.seppuku = Event()
      def start(self):
          self.seppuku.clear()
          self.proc = Thread(target=self._doit)
          self.proc.start()
      def stop(self):
          self.seppuku.set()
          self.proc.join()
      def _doit(self):
          while True:
              self.seppuku.wait(self.object.interval / self.compression_factor)
              if self.seppuku.is_set():
                  break
              self.func(self.args[0], self.args[1])
  
  class day_period(object):
      def __init__(self, object, compression_factor, args=[], kwargs={}):
          self.period = (3600 * 4) / compression_factor
          self.func = object.run
          self.args = args
          self.kwargs = kwargs
          self.seppuku = Event()
      def start(self):
          self.seppuku.clear()
          self.proc = Thread(target=self._doit)
          self.proc.start()
      def stop(self):
          self.seppuku.set()
          self.proc.join()
      def _doit(self):
          while True:
              self.seppuku.wait(self.period)
              if self.seppuku.is_set():
                  break
              self.func(self.args)
  
  class intra_day_traders_mng(object):
      def __init__(self, simulation_config):
          self.config = simulation_config
          self.agents_list = []
          self.agents_dict = {}
          self.p_list = []
          self.compression_factor = simulation_config['simulation_config']['compression_factor']
          self.trader_creator()
          self.first_time = True
          self.day_of_simulation  = simulation_config['simulation_config']['day_number']
  
      def trader_creator(self):
          for agent_name in self.config['agents']['intra_day']:
              for config in self.config['agents']['intra_day'][agent_name]:
                  if agent_name == 'nonclassified_trader':
                      for k in range(config['n_traders']):
                          self.agents_list.append(NON_CLASSIFIED_TRADER_INTRADAY(config))
                          time.sleep(.1)
          for agent_name in self.config['agents']['daily']:
              for config in self.config['agents']['daily'][agent_name]:
                  if agent_name == 'nonclassified_trader':
                      for k in range(config['n_traders']):
                          self.agents_list.append(NON_CLASSIFIED_TRADER_DAILY(config))
                          time.sleep(0.1)
                  if agent_name == "market_maker_trader":
                      for k in range(config['n_traders']):
                          self.agents_list.append(MARKET_MAKER_TRADER_DAILY(config))
                          time.sleep(0.1)
          for agent in self.agents_list:
              self.agents_dict.update({agent.id: agent})
          for agent in self.agents_list:
              agent.set_trader_dict(self.agents_dict)
  
      def random_initial(self):
          agents_random_list = random.choices(self.agents_list, k=len(self.agents_list))
          return agents_random_list
  
      def run(self, args):
          lob = args[0]
          reporter_obj = args[1]
          # when the trader running for first time
          if self.first_time == True:
              lob.time_obj.reset()
              agents_random_list = self.random_initial()
              for agent in agents_random_list:
                  self.p_list.append(Periodic(agent, self.compression_factor, args=(lob,reporter_obj)))
                  self.p_list[-1].start()
                  time.sleep(.1)
              self.first_time = False
          else:
              for proc in self.p_list:
                  proc.stop()
              for agent in self.agents_list:
                  agent.reset_trader(lob)
              time_series = lob.ohcl()
              if len(time_series) == self.day_of_simulation :
                  out = {'out':time_series}
                  with open('output.json', 'w') as outfile:
                      json.dump(out, outfile)
                  reporter_obj.save_as_csv()
                  trade_summary = lob.trade_summary()
                  with open('trade_report.csv', 'w') as csvFile:
                      writer = csv.writer(csvFile)
                      writer.writerows(trade_summary)
                  csvFile.close()
                  sys.exit()
              print("***********************************************************************************")
              print("day is:",lob.time_obj.day)
              lob.time_obj.reset()
              for proc in self.p_list:
                  proc.start()
                  time.sleep(.1)
  
  if __name__ == '__main__':
      with open('config.json', 'r') as f:
          simulation_config = json.load(f)
      intra_day_mng_obj = intra_day_traders_mng(simulation_config)
      reporter_obj = REPORTER()
      # for synchronization of time
      time_obj = TIME_MANAGE(compression_factor=simulation_config['simulation_config']['compression_factor'])
      lob = OrderBook(time_obj, tick_size=simulation_config['simulation_config']['tickSize'])
      day_period(intra_day_mng_obj, simulation_config['simulation_config']['compression_factor'], args=(lob,reporter_obj)).start()
  </code></pre>
  
  <p>And finally the "OrderBook" that defines "self.tape" in the below code:</p>
  
  <pre><code>class OrderBook():
      def __init__(self, time_obj, tick_size=0.0001):
          self.tape = deque(maxlen=None)  # Index [0] is most recent trade
          self.bids = OrderTree()
          self.asks = OrderTree()
          self.lastTick = None
          self.lastTimestamp = 0
          self.tickSize = tick_size
          self.time = 0
          self.nextQuoteID = 0
          self.time_series = []
          self.time_obj = time_obj
  
      def clipPrice(self, price):
          return round(price, int(math.log10(1 / self.tickSize)))
  
      def updateTime(self):
          self.time = int(self.time_obj.now()['time'])
  
      def processOrder(self, quote, fromData, verbose):
          orderType = quote['type']
          orderInBook = None
          if fromData:
              self.time = quote['timestamp']
          else:
              self.updateTime()
              quote['timestamp'] = self.time
          if quote['qty'] &lt;= 0:
              sys.exit('processLimitOrder() given order of qty &lt;= 0')
          if not fromData: self.nextQuoteID += 1
          if orderType == 'market':
              trades = self.processMarketOrder(quote, verbose)
          elif orderType == 'limit':
              quote['price'] = self.clipPrice(quote['price'])
              trades, orderInBook = self.processLimitOrder(quote, fromData, verbose)
          else:
              sys.exit("processOrder() given neither 'market' nor 'limit'")
          return trades, orderInBook
  
      def processOrderList(self, side, orderlist,
                           qtyStillToTrade, quote, verbose):
          trades = []
          qtyToTrade = qtyStillToTrade
          while len(orderlist) &gt; 0 and qtyToTrade &gt; 0:
              headOrder = orderlist.getHeadOrder()
              tradedPrice = headOrder.price
              counterparty = headOrder.tid
              if qtyToTrade &lt; headOrder.qty:
                  tradedQty = qtyToTrade
                  newBookQty = headOrder.qty - qtyToTrade
                  headOrder.updateQty(newBookQty, headOrder.timestamp)
                  qtyToTrade = 0
              elif qtyToTrade == headOrder.qty:
                  tradedQty = qtyToTrade
                  if side == 'bid':
                      self.bids.removeOrderById(headOrder.idNum)
                  else:
                      self.asks.removeOrderById(headOrder.idNum)
                  qtyToTrade = 0
              else:
                  tradedQty = headOrder.qty
                  if side == 'bid':
                      self.bids.removeOrderById(headOrder.idNum)
                  else:
                      self.asks.removeOrderById(headOrder.idNum)
                  qtyToTrade -= tradedQty
              if verbose: print('&gt;&gt;&gt; TRADE \nt=%d $%f n=%d p1=%d p2=%d' %
                                (self.time, tradedPrice, tradedQty,
                                 counterparty, quote['tid']))
  
              transactionRecord = {'timestamp': self.time,
                                   'price': tradedPrice,
                                   'qty': tradedQty,
                                   'time': self.time,
                                   'day': self.time_obj.now()['day']}
              if side == 'bid':
                  transactionRecord['party1'] = [counterparty,
                                                 'bid',
                                                 headOrder.idNum]
                  transactionRecord['party2'] = [quote['tid'],
                                                 'ask',
                                                 None]
              else:
                  transactionRecord['party1'] = [counterparty,
                                                 'ask',
                                                 headOrder.idNum]
                  transactionRecord['party2'] = [quote['tid'],
                                                 'bid',
                                                 None]
              self.tape.append(transactionRecord)
              trades.append(transactionRecord)
          return qtyToTrade, trades
  
      def processMarketOrder(self, quote, verbose):
          trades = []
          qtyToTrade = quote['qty']
          side = quote['side']
          if side == 'bid':
              while qtyToTrade &gt; 0 and self.asks:
                  bestPriceAsks = self.asks.minPriceList()
                  qtyToTrade, newTrades = self.processOrderList('ask',
                                                                bestPriceAsks,
                                                                qtyToTrade,
                                                                quote, verbose)
                  trades += newTrades
          elif side == 'ask':
              while qtyToTrade &gt; 0 and self.bids:
                  bestPriceBids = self.bids.maxPriceList()
                  qtyToTrade, newTrades = self.processOrderList('bid',
                                                                bestPriceBids,
                                                                qtyToTrade,
                                                                quote, verbose)
                  trades += newTrades
          else:
              sys.exit('processMarketOrder() received neither "bid" nor "ask"')
          return trades
  
      def processLimitOrder(self, quote, fromData, verbose):
          orderInBook = None
          trades = []
          qtyToTrade = quote['qty']
          side = quote['side']
          price = quote['price']
          if side == 'bid':
              while (self.asks and
                     price &gt;= self.asks.minPrice() and
                     qtyToTrade &gt; 0):
                  bestPriceAsks = self.asks.minPriceList()
                  qtyToTrade, newTrades = self.processOrderList('ask',
                                                                bestPriceAsks,
                                                                qtyToTrade,
                                                                quote, verbose)
  
                  trades += newTrades
              if qtyToTrade &gt; 0:
                  if not fromData:
                      quote['idNum'] = self.nextQuoteID
                  quote['qty'] = qtyToTrade
                  self.bids.insertOrder(quote)
                  orderInBook = quote
          elif side == 'ask':
              while (self.bids and
                     price &lt;= self.bids.maxPrice() and
                     qtyToTrade &gt; 0):
                  bestPriceBids = self.bids.maxPriceList()
                  qtyToTrade, newTrades = self.processOrderList('bid',
                                                                bestPriceBids,
                                                                qtyToTrade,
                                                                quote, verbose)
                  trades += newTrades
              if qtyToTrade &gt; 0:
                  if not fromData:
                      quote['idNum'] = self.nextQuoteID
                  quote['qty'] = qtyToTrade
                  self.asks.insertOrder(quote)
                  orderInBook = quote
          else:
              sys.exit('processLimitOrder() given neither bid nor ask')
          return trades, orderInBook
  
      def avg_volume_traded(self):
          if self.tape != None and len(self.tape) &gt; 0:
              num = 0
              total_volume = 0
              for entry in self.tape:
                  total_volume = entry['qty'] + total_volume
                  num += 1
              if num != 0 and total_volume != None:
                  return total_volume, num
          else:
              return None, None
  </code></pre>