Add files using upload-large-folder tool

checkpoints/steps_30000_pytorch_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a78d7dc2ac635e77afdf08c54ce3dd738364e47a087b6bc66d5c31c0c5ffc43
+size 8935435793

config.json

@@ -0,0 +1,115 @@
+{
+  "run_id": "0903_libero_spatial_augsteps_0_wo_flash_attention_wo_augsteps_two_view_action_chunk_16_pretrained_vlm",
+  "run_root_dir": "./playground/Checkpoints",
+  "seed": 42,
+  "trackers": [
+    "jsonl",
+    "wandb"
+  ],
+  "wandb_entity": "michaelyu-1101-fudanuniversity",
+  "wandb_project": "Internvla",
+  "is_debug": false,
+  "framework": {
+    "framework_py": "DinoQFormerACT",
+    "qwenvl": {
+      "base_vlm": "/mnt/phwfile/efm_t/zhuyangkun_tmp_need_del/exp/exp_08_09/manip_sys2_qwen25_3b_onevision_molmo_a0all_refsp20/checkpoint-20000",
+      "attn_implementation": "flash_attention_2",
+      "vl_hidden_dim": 2048
+    },
+    "dino": {
+      "dino_backbone": "dinov2_vitl14"
+    },
+    "layer_qformer": {
+      "qformer_end_layer": 37,
+      "qformer_start_layer": 36,
+      "num_query_tokens": 64,
+      "grad_scale": 0.5
+    },
+    "action_model": {
+      "action_model_type": "DiT-B",
+      "action_hidden_dim": 768,
+      "action_dim": 7,
+      "use_ema": false,
+      "future_action_window_size": 7,
+      "past_action_window_size": 0,
+      "repeated_diffusion_steps": 8
+    },
+    "reduce_in_full_precision": true
+  },
+  "datasets": {
+    "vlm_data": {
+      "dataformat": "llava_json",
+      "dataset_use": "asv2_conversation_en,asv2_detailed_description_en,asv2_region_captioning_en,coco_internvl_longcap_en,coco_karpathy_train_567_en,coco_negative_gpt4o_en,coco_poetry_zh,coco_rem_en_zh,cocorem_exist_yorn_en,cocotextv2_en,cocotextv2_gpt4o_en,okvqa_en,refcoco_grounding_aug_en,refcoco_grounding_en,tallyqa_coco_en,toloka_grounding_aug_en,vqav2_en,vsr_en",
+      "eval_dataset": "aokvqa_cauldron_llava_format",
+      "data_flatten": false,
+      "base_interval": 2,
+      "max_pixels": 50176,
+      "min_pixels": 784,
+      "fix_image_size": [
+        224,
+        224
+      ],
+      "model_max_length": 1024,
+      "model_type": "qwen2.5vl",
+      "per_device_batch_size": 4
+    },
+    "vla_data": {
+      "dataset_py": "lerobot_libero",
+      "data_root_dir": "playground/Datasets/LEROBOT_LIBERO_DATA",
+      "data_mix": "libero_spatial",
+      "action_type": "delta_qpos",
+      "CoT_prompt": "Your task is {instruction}. To identify the key objects for your task. Locate their bounding boxes in [x1,y1,x2,y2] format.",
+      "CoT_answer": "bbox",
+      "default_image_resolution": [
+        3,
+        224,
+        224
+      ],
+      "per_device_batch_size": 16,
+      "load_all_data_for_training": true,
+      "obs": [
+        "image_0"
+      ]
+    }
+  },
+  "trainer": {
+    "epochs": 100,
+    "max_train_steps": 100000,
+    "num_warmup_steps": 5000,
+    "save_interval": 10000,
+    "eval_interval": 1000,
+    "learning_rate": {
+      "base": 2.5e-05
+    },
+    "lr_scheduler_type": "cosine_with_min_lr",
+    "scheduler_specific_kwargs": {
+      "min_lr": 1e-06
+    },
+    "freeze_modules": "",
+    "loss_scale": {
+      "vla": 1.0,
+      "vlm": 0.1
+    },
+    "max_grad_norm": 1.0,
+    "warmup_ratio": 0.1,
+    "weight_decay": 0.0,
+    "logging_frequency": 10,
+    "gradient_clipping": 1.0,
+    "gradient_accumulation_steps": 1,
+    "optimizer": {
+      "name": "AdamW",
+      "betas": [
+        0.9,
+        0.95
+      ],
+      "eps": 1e-08,
+      "weight_decay": 1e-08
+    },
+    "is_resume": false,
+    "resume_epoch": null,
+    "resume_step": null,
+    "enable_gradient_checkpointing": true,
+    "enable_mixed_precision_training": true
+  },
+  "output_dir": "./playground/Checkpoints/0903_libero_spatial_augsteps_0_wo_flash_attention_wo_augsteps_two_view_action_chunk_16_pretrained_vlm"
+}

config.yaml

@@ -0,0 +1,96 @@
+run_id: 0903_libero_spatial_augsteps_0_wo_flash_attention_wo_augsteps_two_view_action_chunk_16_pretrained_vlm
+run_root_dir: ./playground/Checkpoints
+seed: 42
+trackers:
+- jsonl
+- wandb
+wandb_entity: michaelyu-1101-fudanuniversity
+wandb_project: Internvla
+is_debug: false
+framework:
+  framework_py: DinoQFormerACT
+  qwenvl:
+    base_vlm: /mnt/phwfile/efm_t/zhuyangkun_tmp_need_del/exp/exp_08_09/manip_sys2_qwen25_3b_onevision_molmo_a0all_refsp20/checkpoint-20000
+    attn_implementation: flash_attention_2
+    vl_hidden_dim: 2048
+  dino:
+    dino_backbone: dinov2_vitl14
+  layer_qformer:
+    qformer_end_layer: 37
+    qformer_start_layer: 36
+    num_query_tokens: 64
+    grad_scale: 0.5
+  action_model:
+    action_model_type: DiT-B
+    action_hidden_dim: 768
+    action_dim: 7
+    use_ema: false
+    future_action_window_size: 7
+    past_action_window_size: 0
+    repeated_diffusion_steps: 8
+  reduce_in_full_precision: true
+datasets:
+  vlm_data:
+    dataformat: llava_json
+    dataset_use: asv2_conversation_en,asv2_detailed_description_en,asv2_region_captioning_en,coco_internvl_longcap_en,coco_karpathy_train_567_en,coco_negative_gpt4o_en,coco_poetry_zh,coco_rem_en_zh,cocorem_exist_yorn_en,cocotextv2_en,cocotextv2_gpt4o_en,okvqa_en,refcoco_grounding_aug_en,refcoco_grounding_en,tallyqa_coco_en,toloka_grounding_aug_en,vqav2_en,vsr_en
+    eval_dataset: aokvqa_cauldron_llava_format
+    data_flatten: false
+    base_interval: 2
+    max_pixels: 50176
+    min_pixels: 784
+    fix_image_size:
+    - 224
+    - 224
+    model_max_length: 1024
+    model_type: qwen2.5vl
+    per_device_batch_size: 4
+  vla_data:
+    dataset_py: lerobot_libero
+    data_root_dir: playground/Datasets/LEROBOT_LIBERO_DATA
+    data_mix: libero_spatial
+    action_type: delta_qpos
+    CoT_prompt: Your task is {instruction}. To identify the key objects for your task.
+      Locate their bounding boxes in [x1,y1,x2,y2] format.
+    CoT_answer: bbox
+    default_image_resolution:
+    - 3
+    - 224
+    - 224
+    per_device_batch_size: 16
+    load_all_data_for_training: true
+    obs:
+    - image_0
+trainer:
+  epochs: 100
+  max_train_steps: 100000
+  num_warmup_steps: 5000
+  save_interval: 10000
+  eval_interval: 1000
+  learning_rate:
+    base: 2.5e-05
+  lr_scheduler_type: cosine_with_min_lr
+  scheduler_specific_kwargs:
+    min_lr: 1.0e-06
+  freeze_modules: ''
+  loss_scale:
+    vla: 1.0
+    vlm: 0.1
+  max_grad_norm: 1.0
+  warmup_ratio: 0.1
+  weight_decay: 0.0
+  logging_frequency: 10
+  gradient_clipping: 1.0
+  gradient_accumulation_steps: 1
+  optimizer:
+    name: AdamW
+    betas:
+    - 0.9
+    - 0.95
+    eps: 1.0e-08
+    weight_decay: 1.0e-08
+  is_resume: false
+  resume_epoch: null
+  resume_step: null
+  enable_gradient_checkpointing: true
+  enable_mixed_precision_training: true
+output_dir: ./playground/Checkpoints/0903_libero_spatial_augsteps_0_wo_flash_attention_wo_augsteps_two_view_action_chunk_16_pretrained_vlm

dataset_statistics.json

@@ -0,0 +1,133 @@
+{
+  "franka": {
+    "action": {
+      "mean": [
+        0.15312479436397552,
+        0.13707277178764343,
+        -0.15526802837848663,
+        -0.005176450591534376,
+        -0.01120874285697937,
+        -0.020194264128804207,
+        0.4578818082809448
+      ],
+      "std": [
+        0.41272708773612976,
+        0.34724321961402893,
+        0.50869220495224,
+        0.037266165018081665,
+        0.07244449853897095,
+        0.05762382969260216,
+        0.49827873706817627
+      ],
+      "max": [
+        0.9375,
+        0.9375,
+        0.9375,
+        0.1971428543329239,
+        0.33642858266830444,
+        0.375,
+        1.0
+      ],
+      "min": [
+        -0.9375,
+        -0.9375,
+        -0.9375,
+        -0.1875,
+        -0.3675000071525574,
+        -0.36000001430511475,
+        0.0
+      ],
+      "q01": [
+        -0.7454732114076613,
+        -0.6616071462631226,
+        -0.9375,
+        -0.1071428582072258,
+        -0.20678570866584778,
+        -0.1842857152223587,
+        0.0
+      ],
+      "q99": [
+        0.9375,
+        0.8758928775787354,
+        0.9321428537368774,
+        0.1039285734295845,
+        0.17678570747375488,
+        0.14571428298950195,
+        1.0
+      ],
+      "mask": [
+        true,
+        true,
+        true,
+        true,
+        true,
+        true,
+        false
+      ]
+    },
+    "state": {
+      "mean": [
+        -0.024462558329105377,
+        0.106529600918293,
+        1.0580483675003052,
+        3.0628468990325928,
+        -0.10464039444923401,
+        0.08307311683893204,
+        0.01995457336306572,
+        -0.020162804052233696
+      ],
+      "std": [
+        0.1101478561758995,
+        0.13784688711166382,
+        0.10442823916673634,
+        0.10451053828000996,
+        0.4112098217010498,
+        0.2176690548658371,
+        0.017260896041989326,
+        0.0171116404235363
+      ],
+      "max": [
+        0.1759040206670761,
+        0.3904820382595062,
+        1.3290715217590332,
+        3.4566118717193604,
+        1.2268599271774292,
+        1.0429412126541138,
+        0.041053611785173416,
+        0.000775813648942858
+      ],
+      "min": [
+        -0.3095473051071167,
+        -0.29250794649124146,
+        0.9095591306686401,
+        2.497488260269165,
+        -1.8006486892700195,
+        -0.7207611203193665,
+        -0.0004703797458205372,
+        -0.041536275297403336
+      ],
+      "q01": [
+        -0.2727657300233841,
+        -0.23721413239836692,
+        0.9160063165426254,
+        2.77949666261673,
+        -1.3187511622905732,
+        -0.41989982962608335,
+        0.001503719249740243,
+        -0.03989770736545324
+      ],
+      "q99": [
+        0.13529365032911292,
+        0.3629165390133857,
+        1.2862326657772063,
+        3.2829698753356933,
+        0.9332760351896285,
+        0.6325724506378171,
+        0.039933966137468815,
+        -0.001671919699292631
+      ]
+    },
+    "num_transitions": 52970,
+    "num_trajectories": 432
+  }
+}

run_lerobot_datasets.sh

@@ -0,0 +1,64 @@
+export HF_HOME=/mnt/petrelfs/share/yejinhui/Models/huggingface_cache
+
+export NCCL_SOCKET_IFNAME=bond0
+export NCCL_IB_HCA=mlx5_2,mlx5_3
+
+# 用于check save 的时候的通信
+export NCCL_BLOCKING_WAIT=1
+export NCCL_ASYNC_ERROR_HANDLING=1
+export NCCL_TIMEOUT=1000  # 超时时间设为 1 小时（单位：秒）
+
+cd /mnt/petrelfs/yujunqiu/code/vla-baseline/llavavla-00hf1
+
+# MODEL_PATH=/mnt/petrelfs/yejinhui/Projects/llavavla/playground/Pretrained_models/Qwen2.5-VL-3B-Instruct # must be a local path, due to simpler will run in other where
+# data_root_dir=./playground/Datasets/OXE_LEROBOT_DATASET
+run_root_dir=./playground/Checkpoints
+task_name=libero_spatial
+run_id=0903_${task_name}_augsteps_0_wo_flash_attention_wo_augsteps_two_view_action_chunk_16_pretrained_vlm
+
+
+export WANDB_MODE=disabled
+
+output_dir=${run_root_dir}/${run_id}
+mkdir -p ${output_dir}
+# mv this script to the output dir
+cp $0 ${output_dir}/
+
+  # --pretrained_checkpoint ${MODEL_PATH} \
+# export CUDA_VISIBLE_DEVICES=4,5,6,7
+
+  # --datasets.vla_data.data_mix libero_goal \
+  # --framework.framework_py qwenpi \
+
+DEBUG=False
+# DEBUG=True
+
+if [ "$DEBUG" = True ]; then
+  num_processes=1
+  run_id=debug
+else
+  num_processes=8
+fi
+
+
+accelerate launch \
+  --config_file scripts/run_scripts/deepspeed_zero2.yaml \
+  --num_processes ${num_processes} \
+  llavavla/training/train_qwenvla.py \
+  --config_yaml ./llavavla/config/lerobot_data/qwenvla_cotrain_libero.yaml \
+  --datasets.vla_data.per_device_batch_size 16 \
+  --datasets.vla_data.data_mix ${task_name} \
+  --framework.action_model.future_action_window_size 7 \
+  --trainer.max_train_steps 100_000 \
+  --trainer.save_interval 10_000 \
+  --run_root_dir ${run_root_dir} \
+  --run_id ${run_id} \
+  --wandb_project Internvla \
+  --wandb_entity michaelyu-1101-fudanuniversity \
+  --is_debug ${DEBUG} \
+  --framework.qwenvl.base_vlm /mnt/phwfile/efm_t/zhuyangkun_tmp_need_del/exp/exp_08_09/manip_sys2_qwen25_3b_onevision_molmo_a0all_refsp20/checkpoint-20000
+
+#   --framework.qwenvl.base_vlm ${MODEL_PATH} \
+#   --data_root_dir ${data_root_dir} \
+
+  # --is_debug True

summary.jsonl

@@ -0,0 +1,3 @@
+{"steps": 10000}
+{"steps": 20000}
+{"steps": 30000}