def evaluate(env, policy, num_episodes = 10):
  """Evaluates the policy.

  Args:
    env: Environment to evaluate the policy on.
    policy: Policy to evaluate.
    num_episodes: A number of episodes to average the policy on.

  Returns:
    Averaged reward and a total number of steps.
  """
  total_timesteps = 0
  total_returns = 0.0

  for _ in range(num_episodes):
    episode_return = 0
    timestep = env.reset()

    while not timestep.is_last():
      action = policy.act(timestep.observation)
      timestep = env.step(action)

      total_returns += timestep.reward[0]
      episode_return += timestep.reward[0]
      total_timesteps += 1

  return total_returns / num_episodes, total_timesteps / num_episodes