@gen.coroutine
  def _FailOp(self, op, type, value, tb, initial_backoff_secs=_INITIAL_BACKOFF_SECS):
    """Writes the failure to the log and puts the operation to sleep in the database with
    a backoff. The operation will get re-run once the backoff expires. If the operation has
    failed less than 3 times, then the next operation will *not* be run until this operation
    has been retried at least 3 times. That many retries indicate there's probably a real
    issue, so the operation is considered to be in "quarantine", and execution will proceed
    to the next operation.

    TODO(Andy): Recognize certain exceptions as disallowing quarantine. In that case, we will
                not proceed to the next operation. Instead, we will abandon the lock, which
                will block further user actions until the operation is fixed.
    """
    elapsed_secs = time.time() - op.timestamp
    logging.error('FAILURE: user: %d, device: %d, op: %s, method: %s in %.3fs, %s' %
                  (op.user_id, op.device_id, op.operation_id, op.method, elapsed_secs, value),
                  exc_info=(type, value, tb))
    exc = ''.join(traceback.format_exception(type, value, tb))
    if op.first_failure:
      op.last_failure = exc
    else:
      op.first_failure = exc

    # Truncate stack traces if they make the operation too large.
    max_traceback_size = UserOpManager._MAX_OPERATION_SIZE - len(op.json)
    if (len(op.first_failure or '') + len(op.last_failure or '')) > max_traceback_size:
      if op.first_failure and op.last_failure:
        max_traceback_size = int(max_traceback_size / 2)
      if op.first_failure:
        op.first_failure = op.first_failure[-max_traceback_size:]
      if op.last_failure:
        op.last_failure = op.last_failure[-max_traceback_size:]

    # Compute an exponential backoff based on number of attempts.
    op.backoff = time.time() + (1 << min(op.attempts, UserOpManager._MAX_BACKOFF_STEPS)) * initial_backoff_secs
    retry_after_secs = op.backoff - time.time()

    op.attempts = op.attempts + 1
    if op.attempts < 3:
      # Less than 3 attempts of this operation, so hold lock and try again after a delay.
      logging.warning('%d failed attempt%s of operation %s for user %d; hold lock and retry in %.2fs' %
                      (op.attempts, util.Pluralize(op.attempts), op.operation_id, op.user_id, retry_after_secs))
    else:
      # The operation failed at least 3 times, so just write it back to database in quarantine
      # and move to next operation.
      logging.warning('%d failed attempt%s of operation %s for user %d; put operation into quarantine and '
                      'move to next operation; retry operation no sooner than %.2fs from now' %
                      (op.attempts, util.Pluralize(op.attempts), op.operation_id, op.user_id, retry_after_secs))
      self._last_op_id = op.operation_id
      op.quarantine = 1

    yield gen.Task(op.Update, self._client)
dataset/ETHPy150Open viewfinderco/viewfinder/backend/op/user_op_manager.py/UserOpManager._FailOp/original