def gen_updates_adadelta(loss, all_parameters,
                         learning_rate=1.0,
                         rho=0.95,
                         epsilon=1e-6):
    """
    in the paper, no learning rate is considered (so learning_rate=1.0).
    Probably best to keep it at this value. epsilon is important for the very
    first update (so the numerator does not become 0).

    rho = 0.95 and epsilon=1e-6 are suggested in the paper and reported to work
    for multiple datasets (MNIST, speech).

    see "Adadelta: an adaptive learning rate method" by Matthew Zeiler for more
    info.
    """
    all_grads = [theano.grad(loss, param) for param in all_parameters]
    # initialise to zeroes with the right shape
    all_accumulators = [theano.shared(param.get_value() * 0.)
                        for param in all_parameters]
    all_delta_accumulators = [theano.shared(param.get_value() * 0.)
                              for param in all_parameters]

    # all_accumulators: accumulate gradient magnitudes
    # all_delta_accumulators: accumulate update magnitudes (recursive!)

    updates = []
    for param_i, grad_i, acc_i, acc_delta_i in zip(all_parameters, all_grads,
                                                   all_accumulators,
                                                   all_delta_accumulators):
        acc_i_new = rho * acc_i + (1 - rho) * grad_i ** 2
        updates.append((acc_i, acc_i_new))

        update_i = (grad_i * T.sqrt(acc_delta_i + epsilon) /
                    T.sqrt(acc_i_new + epsilon))
        updates.append((param_i, learning_rate * update_i - param_i))

        acc_delta_i_new = rho * acc_delta_i + (1 - rho) * update_i ** 2
        updates.append((acc_delta_i, acc_delta_i_new))

    return updates
dataset/ETHPy150Open ifp-uiuc/anna/anna/layers/layers.py/gen_updates_adadelta/swapped operands of `-`