def gen_updates_adagrad(loss, all_parameters, learning_rate=1.0, epsilon=1e-6):
    """
    epsilon is not included in the typical formula,

    See "Notes on AdaGrad" by Chris Dyer for more info.
    """
    all_grads = [theano.grad(loss, param) for param in all_parameters]
    # initialise to zeroes with the right shape
    all_accumulators = [theano.shared(param.get_value() * 0.)
                        for param in all_parameters]

    updates = []
    for param_i, grad_i, acc_i in zip(all_parameters, all_grads,
                                      all_accumulators):
        acc_i_new = acc_i + grad_i ** 2
        updates.append((acc_i, acc_i_new))
        updates.append(
            (param_i,
             learning_rate * grad_i / T.sqrt(acc_i_new + epsilon) - param_i))

    return updates
dataset/ETHPy150Open ifp-uiuc/anna/anna/layers/layers.py/gen_updates_adagrad/swapped operands of `-`