title: how to get reproducible result in Tensorflow
body: <p>I built 5-layer neural network by using tensorflow. </p>
  
  <p>I have a problem to get reproducible results (or stable results). </p>
  
  <p>I found similar questions regarding reproducibility of tensorflow and the corresponding answers, such as <a href="https://stackoverflow.com/questions/36288235/how-to-get-stable-results-with-tensorflow-setting-random-seed?rq=1">How to get stable results with TensorFlow, setting random seed</a></p>
  
  <p>But the problem is not solved yet. </p>
  
  <p>I also set random seed like the following</p>
  
  <pre><code>tf.set_random_seed(1)
  </code></pre>
  
  <p>Furthermore, I added seed options to every random function such as</p>
  
  <pre><code>b1 = tf.Variable(tf.random_normal([nHidden1], seed=1234))
  </code></pre>
  
  <p>I confirmed that the first epoch shows the identical results, but not identical from the second epoch little by little.</p>
  
  <p>How can I get the reproducible results? </p>
  
  <p>Am I missing something?</p>
  
  <p>Here is a code block I use. </p>
  
  <pre><code>def xavier_init(n_inputs, n_outputs, uniform=True):
      if uniform:
          init_range = tf.sqrt(6.0 / (n_inputs + n_outputs))
          return tf.random_uniform_initializer(-init_range, init_range, seed=1234)
      else:
          stddev = tf.sqrt(3.0 / (n_inputs + n_outputs))
          return tf.truncated_normal_initializer(stddev=stddev, seed=1234)
  
  
  import numpy as np
  import tensorflow as tf
  import dataSetup
  from scipy.stats.stats import pearsonr
  
  tf.set_random_seed(1)
  
  x_train, y_train, x_test, y_test = dataSetup.input_data()
  
  # Parameters
  learningRate = 0.01
  trainingEpochs = 1000000
  batchSize = 64 
  displayStep = 100
  thresholdReduce = 1e-6
  thresholdNow = 0.6
  #dropoutRate = tf.constant(0.7)
  
  
  # Network Parameter
  nHidden1 = 128 # number of 1st layer nodes
  nHidden2 = 64 # number of 2nd layer nodes
  nInput = 24 #
  nOutput = 1 # Predicted score: 1 output for regression
  
  # save parameter
  modelPath = 'model/model_layer5_%d_%d_mini%d_lr%.3f_noDrop_rollBack.ckpt' %(nHidden1, nHidden2, batchSize, learningRate)
  
  # tf Graph input
  X = tf.placeholder("float", [None, nInput])
  Y = tf.placeholder("float", [None, nOutput])
  
  # Weight
  W1 = tf.get_variable("W1", shape=[nInput, nHidden1], initializer=xavier_init(nInput, nHidden1))
  W2 = tf.get_variable("W2", shape=[nHidden1, nHidden2], initializer=xavier_init(nHidden1, nHidden2))
  W3 = tf.get_variable("W3", shape=[nHidden2, nHidden2], initializer=xavier_init(nHidden2, nHidden2))
  W4 = tf.get_variable("W4", shape=[nHidden2, nHidden2], initializer=xavier_init(nHidden2, nHidden2))
  WFinal = tf.get_variable("WFinal", shape=[nHidden2, nOutput], initializer=xavier_init(nHidden2, nOutput))
  
  # biases
  b1 = tf.Variable(tf.random_normal([nHidden1], seed=1234))
  b2 = tf.Variable(tf.random_normal([nHidden2], seed=1234))
  b3 = tf.Variable(tf.random_normal([nHidden2], seed=1234))
  b4 = tf.Variable(tf.random_normal([nHidden2], seed=1234))
  bFinal = tf.Variable(tf.random_normal([nOutput], seed=1234))
  
  # Layers for dropout
  L1 = tf.nn.relu(tf.add(tf.matmul(X, W1), b1))
  L2 = tf.nn.relu(tf.add(tf.matmul(L1, W2), b2))
  L3 = tf.nn.relu(tf.add(tf.matmul(L2, W3), b3))
  L4 = tf.nn.relu(tf.add(tf.matmul(L3, W4), b4))
  
  hypothesis = tf.add(tf.matmul(L4, WFinal), bFinal)
  print "Layer setting DONE..."
  
  # define loss and optimizer
  cost = tf.reduce_mean(tf.square(hypothesis - Y))
  optimizer = tf.train.AdamOptimizer(learning_rate=learningRate).minimize(cost)
  
  # Initialize the variable
  init = tf.initialize_all_variables()
  
  # save op to save and restore all the variables
  saver = tf.train.Saver()
  
  with tf.Session() as sess:
      # initialize
      sess.run(init)
      print "Initialize DONE..."
  
      # Training
      costPrevious = 100000000000000.0
      best = float("INF")
  
      totalBatch = int(len(x_train)/batchSize)
      print "Total Batch: %d" %totalBatch
  
      for epoch in range(trainingEpochs):
          #print "EPOCH: %04d" %epoch
          avgCost = 0.
  
          for i in range(totalBatch):
              np.random.seed(i+epoch)
              randidx = np.random.randint(len(x_train), size=batchSize)
              batch_xs = x_train[randidx,:]
              batch_ys = y_train[randidx,:]
  
              # Fit traiing using batch data
              sess.run(optimizer, feed_dict={X:batch_xs, Y:batch_ys})
  
              # compute average loss
              avgCost += sess.run(cost, feed_dict={X:batch_xs, Y:batch_ys})/totalBatch
  
          # compare the current cost and the previous
          # if current cost &gt; the previous
          # just continue and make the learning rate half
  
          #print "Cost: %1.8f --&gt; %1.8f at epoch %05d" %(costPrevious, avgCost, epoch+1)
  
          if avgCost &gt; costPrevious + .5:
              #sess.run(init)
              load_path = saver.restore(sess, modelPath)
              print "Cost increases at the epoch %05d" %(epoch+1)
              print "Cost: %1.8f --&gt; %1.8f" %(costPrevious, avgCost)
              continue
  
          costNow = avgCost
          reduceCost = abs(costPrevious - costNow)
          costPrevious = costNow
  
          #Display logs per epoch step
          if costNow &lt; best:
              best = costNow
              bestMatch = sess.run(hypothesis, feed_dict={X:x_test})
              # model save
              save_path = saver.save(sess, modelPath)
  
          if epoch % displayStep == 0:
              print "step {}".format(epoch)
              pearson = np.corrcoef(bestMatch.flatten(), y_test.flatten())
              print 'train loss = {}, current loss = {}, test corrcoef={}'.format(best, costNow, pearson[0][1])
  
          if reduceCost &lt; thresholdReduce or costNow &lt; thresholdNow:
              print "Epoch: %04d, Cost: %.9f, Prev: %.9f, Reduce: %.9f" %(epoch+1, costNow, costPrevious, reduceCost)
              break
  
      print "Optimization Finished"
  </code></pre>