| { | |
| "robovlm_name": "RoboPiZero", | |
| "parent": null, | |
| "task_name": "pizero_finetune", | |
| "model": "pizero", | |
| "model_url": "https://huggingface.co/google/paligemma-3b-pt-224", | |
| "seq_len": 1, | |
| "image_size": 224, | |
| "image_mean": [ | |
| 0.48145466, | |
| 0.4578275, | |
| 0.40821073 | |
| ], | |
| "image_std": [ | |
| 0.26862954, | |
| 0.26130258, | |
| 0.27577711 | |
| ], | |
| "window_size": 1, | |
| "fwd_pred_next_n": 4, | |
| "arm_gripper_loss_ratio": 0.01, | |
| "text_loss_ratio": 0.1, | |
| "cap_loss_ratio": 0.05, | |
| "fwd_loss_ratio": 0, | |
| "vl_cotrain_ratio": 1, | |
| "seed": 123, | |
| "batch_size": 4, | |
| "num_workers": 16, | |
| "data_scale": 1, | |
| "optimizer": "adam", | |
| "learning_rate": 2e-05, | |
| "min_lr_scale": 0.01, | |
| "weight_decay": 0, | |
| "warmup_epochs": 0.25, | |
| "warmup_steps": 0, | |
| "warmup_ratio": null, | |
| "use_hand_rgb": true, | |
| "use_time_causal_attn": false, | |
| "use_mim_obs_loss": false, | |
| "use_pixel_loss": true, | |
| "use_obs_queries": true, | |
| "use_vision_resampler": false, | |
| "vision_masked_ratio": 0.9, | |
| "use_tube_mask": false, | |
| "cache_root": "runs/cache/pizero", | |
| "model_load_path": null, | |
| "model_load_source": "torch", | |
| "resume": null, | |
| "use_cot_stage_token": false, | |
| "use_cot": false, | |
| "force_model_cot": false, | |
| "use_chat_stage_token": false, | |
| "cot_tags": [ | |
| "task", | |
| "plan", | |
| "bboxes", | |
| "subtask_reason", | |
| "subtask", | |
| "move_reason", | |
| "move", | |
| "gripper" | |
| ], | |
| "model_path": "/share/project/lxh/project/VLMs/paligemma-3b-pt-224", | |
| "model_config": "/share/project/lxh/project/VLMs/paligemma-3b-pt-224", | |
| "deepspeed_config": { | |
| "fp16": { | |
| "enabled": false, | |
| "loss_scale": 0, | |
| "loss_scale_window": 1000, | |
| "initial_scale_power": 16, | |
| "hysteresis": 2, | |
| "min_loss_scale": 1 | |
| }, | |
| "bf16": { | |
| "enabled": true | |
| }, | |
| "zero_allow_untested_optimizer": true, | |
| "train_micro_batch_size_per_gpu": 4, | |
| "zero_optimization": { | |
| "stage": 2, | |
| "overlap_comm": true, | |
| "contiguous_gradients": true, | |
| "reduce_bucket_size": 500000000.0, | |
| "allgather_partitions": true, | |
| "allgather_bucket_size": 500000000.0, | |
| "reduce_scatter": true | |
| }, | |
| "communication": { | |
| "nccl_timeout": 3600, | |
| "all_reduce_timeout": 7200 | |
| }, | |
| "gradient_accumulation_steps": 4, | |
| "gradient_clipping": 0.0 | |
| }, | |
| "train_setup": { | |
| "precision": "bf16", | |
| "predict_action": true, | |
| "predict_forward": false, | |
| "predict_forward_hand": false, | |
| "predict_caption": false, | |
| "train_vision": true, | |
| "bits": -1, | |
| "freeze_mm_mlp_adapter": false, | |
| "freeze_backbone": false, | |
| "freeze_resampler": false, | |
| "tune_mm_mlp_adapter": false, | |
| "mm_use_im_start_end": false, | |
| "mm_use_im_patch_token": false, | |
| "gradient_checkpointing": false, | |
| "lora_enable": false, | |
| "mm_projector_lr": 0.0001, | |
| "lora_r": 64, | |
| "lora_alpha": 16, | |
| "lora_dropout": 0.05, | |
| "lora_bias": "none", | |
| "train_text_embedding": true | |
| }, | |
| "vision_resampler": { | |
| "vis_dim": 1024, | |
| "depth": 8, | |
| "dim_head": 64, | |
| "heads": 8, | |
| "num_latents": 64 | |
| }, | |
| "act_encoder": null, | |
| "act_head": { | |
| "type": "FCDecoder", | |
| "hidden_size": 1024, | |
| "action_dim": 7, | |
| "down_sample": "none", | |
| "latent": 1, | |
| "fwd_pred_next_n": 1, | |
| "window_size": 1, | |
| "action_space": "continuous", | |
| "with_history": true, | |
| "history_type": "post", | |
| "n_bin": 256, | |
| "min_action": -1, | |
| "max_action": 1 | |
| }, | |
| "fwd_head": null, | |
| "tokenizer": { | |
| "type": "AutoProcessor", | |
| "pretrained_model_name_or_path": "/share/project/lxh/project/VLMs/paligemma-3b-pt-224", | |
| "tokenizer_type": "paligemma", | |
| "max_text_len": 256, | |
| "additional_special_tokens": null | |
| }, | |
| "vlm": { | |
| "type": "Pizero", | |
| "name": "pizero", | |
| "dtype": "bfloat16", | |
| "pretrained_model_name_or_path": "/share/project/lxh/project/VLMs/paligemma-3b-pt-224", | |
| "mixture": { | |
| "vlm": { | |
| "hidden_size": 2048, | |
| "intermediate_size": 16384, | |
| "use_final_norm": false, | |
| "cache": true, | |
| "rope_theta": 10000.0, | |
| "use_quantize": false, | |
| "use_lora": false, | |
| "adaptive_mode": null | |
| }, | |
| "proprio": { | |
| "hidden_size": 1024, | |
| "intermediate_size": 4096, | |
| "use_final_norm": true, | |
| "cache": true, | |
| "rope_theta": 100.0, | |
| "use_quantize": false, | |
| "use_lora": false, | |
| "adaptive_mode": null | |
| }, | |
| "action": { | |
| "hidden_size": 1024, | |
| "intermediate_size": 4096, | |
| "use_final_norm": true, | |
| "cache": false, | |
| "rope_theta": 100.0, | |
| "use_quantize": false, | |
| "use_lora": false, | |
| "adaptive_mode": null | |
| } | |
| }, | |
| "num_hidden_layers": 18, | |
| "num_attention_heads": 8, | |
| "num_key_value_heads": 1, | |
| "num_inference_steps": 10, | |
| "head_dim": 256, | |
| "rms_norm_eps": 1e-06, | |
| "attention_bias": false, | |
| "attention_dropout": 0.0, | |
| "max_image_text_tokens": 276, | |
| "cond_steps": 1, | |
| "horizon_steps": 4, | |
| "pad_token_id": 0, | |
| "time_hidden_size": 256, | |
| "action_expert_adaptive_mode": null, | |
| "final_action_clip_value": 1.0, | |
| "action_dim": 7, | |
| "proprio_dim": 7, | |
| "time_max_period": 100.0, | |
| "flow_sampling": "beta", | |
| "flow_alpha": 1.5, | |
| "flow_beta": 1.0, | |
| "flow_sig_min": 0.001 | |
| }, | |
| "trainer": { | |
| "accelerator": "gpu", | |
| "strategy": "deepspeed_stage_2", | |
| "precision": "bf16", | |
| "logger": [ | |
| "tensorboard" | |
| ], | |
| "gradient_clip_val": 1.0, | |
| "use_distributed_sampler": false, | |
| "log_every_n_steps": 10, | |
| "max_epochs": 10, | |
| "val_check_interval": null, | |
| "check_val_every_n_epoch": 1, | |
| "max_steps": -1, | |
| "accumulate_grad_batches": 4 | |
| }, | |
| "use_cot_data": true, | |
| "train_dataset": { | |
| "type": "OpenVLADatasetByRank", | |
| "data_root_dir": "/share/project/lxh/datasets/tf_datasets", | |
| "model_name": "paligemma", | |
| "image_aug": true, | |
| "mode": "train", | |
| "data_mix": "bridge", | |
| "window_sample": "sliding", | |
| "organize_type": "interleave", | |
| "shuffle_buffer_size": 51200, | |
| "train": true | |
| }, | |
| "val_dataset": { | |
| "type": "OpenVLADatasetByRank", | |
| "data_root_dir": "/share/project/lxh/datasets/tf_datasets", | |
| "model_name": "paligemma", | |
| "mode": "train", | |
| "data_mix": "bridge", | |
| "window_sample": "sliding", | |
| "organize_type": "interleave", | |
| "shuffle_buffer_size": 10000, | |
| "train": false | |
| }, | |
| "raw_config_path": "/share/project/lxh/project/RoboVLMs-dev/configs/baai_oxe/finetune_pizero_bridge_no-cot.json", | |
| "config": "/share/project/lxh/project/RoboVLMs-dev/configs/baai_oxe/finetune_pizero_bridge_no-cot.json", | |
| "gpus": 8, | |
| "num_nodes": 4, | |
| "log_dir": "runs/logs/pizero/pizero_finetune/2025-04-29/05-54", | |
| "output_dir": "runs/checkpoints/pizero/pizero_finetune/2025-04-29/05-54", | |
| "data_dir": null, | |
| "annotation_file": null, | |
| "data_subfolder": null, | |
| "task_num": null, | |
| "exp_name": "05-54", | |
| "use_multi_modal_emb": false, | |
| "no_video_pretrained_model": false, | |
| "finetune": false, | |
| "llm": { | |
| "type": null, | |
| "n_embd": null, | |
| "n_layer": null, | |
| "n_head": null | |
| } | |
| } |