def import_iterator(iterf, dbmethods):
  """
  """
  # this function could dynamically increase or decrease the block
  rowiter = iterf()
  types = iterf.types
  blocksize = 100000
  buf = []

  if iterf.header_inferred:
    rowiter.next()

  rowidx = dbmethods.get_max_id() or 0

  for row in rowiter:
    rowidx += 1

    row = transform_and_validate(types, row)

    if iterf.add_id_col:
      row.append(str(rowidx))

    if row is not None and len(row) == len(iterf.types):
      buf.append(row)
    elif row is not None and len(row) != len(iterf.types):
      print >>dbmethods.errfile, ','.join(map(to_utf, row))

    if len(buf) > 0 and len(buf) % blocksize == 0:
      try:
        _log.info("flushing %s rows", len(buf))
        success = dbmethods.import_block(buf, iterf)
      except Exception as e:
        _log.warn(traceback.format_exc())
      buf = []

  if len(buf) > 0:
    try:
      success = dbmethods.import_block(buf, iterf)
      _log.info( "loaded\t%s\t%d", success, rowidx )
    except Exception as e:
      _log.warn(traceback.format_exc())
dataset/ETHPy150Open sirrice/dbtruck/dbtruck/dbtruck.py/import_iterator/original