FineTune ASR with freeze encoder
Hello!

I try FineTune ASR with freeze encoder on Russian language.

I am taking the parameters of the optimizer, learning rate from the this paper
https://arxiv.org/pdf/2005.04290.pdf and i set warmup_steps the value to 2000, but when training the model, the loss value is "buried", and WER is in one. 
What actions to take to solve this problem?

model_weights.ckpt is unpack stt_en_jasper10x5dr.nemo model
code for training:
```
@hydra_runner(config_path="./", config_name="jasper_10x5dr.yaml")
def main(cfg):
    logging.info(f'Hydra config: {OmegaConf.to_yaml(cfg)}')

    trainer = pl.Trainer(**cfg.trainer, amp_level='O1')
    exp_manager(trainer, cfg.get("exp_manager", None))
    asr_model = EncDecCTCModel(cfg=cfg.model, trainer=trainer)
    # Initialize the weights of the model from another model, if provided via config
    
    state_dict = torch.load('model_weights.ckpt' , map_location='cpu')
    encoder_state_dict = {key[8:] : value for key,value in state_dict.items() if key.split('.')[0] == 'encoder'}
    asr_model.encoder.load_state_dict(encoder_state_dict)
    del state_dict
    del encoder_state_dict
    torch.cuda.empty_cache()
    asr_model.encoder.freeze()

    trainer.fit(asr_model)

    if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.manifest_filepath is not None:
        gpu = 1 if cfg.trainer.gpus != 0 else 0
        test_trainer = pl.Trainer(
            gpus=gpu,
            precision=trainer.precision,
            amp_level=trainer.accelerator_connector.amp_level,
            amp_backend=cfg.trainer.get("amp_backend", "native"),
        )
        if asr_model.prepare_test(test_trainer):
            test_trainer.test(asr_model)
if __name__ == '__main__':
    main()

```