Add files using upload-large-folder tool

analyze_accuracy.ipynb

@@ -0,0 +1,1015 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "89f2b537",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import re\n",
+    "from typing import Dict, List, Optional\n",
+    "from mathruler.grader import extract_boxed_content, grade_answer\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "8590ec56",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "from pathlib import Path\n",
+    "from typing import List, Dict, Union\n",
+    "from typing import Dict, List, Any\n",
+    "import re\n",
+    "from typing import List\n",
+    "\n",
+    "def read_json(path: Union[str, Path]) -> List[Dict]:\n",
+    "    \"\"\"\n",
+    "    Read a JSON file and return its contents as a list of dicts.\n",
+    "\n",
+    "    Parameters\n",
+    "    ----------\n",
+    "    path : str or Path\n",
+    "        Path to a JSON file whose root is a JSON array.\n",
+    "\n",
+    "    Returns\n",
+    "    -------\n",
+    "    List[Dict]\n",
+    "        Each element of the top-level JSON array, parsed into a Python dict.\n",
+    "\n",
+    "    Raises\n",
+    "    ------\n",
+    "    ValueError\n",
+    "        If the JSON root is not a list.\n",
+    "    json.JSONDecodeError\n",
+    "        If the file is not valid JSON.\n",
+    "    \"\"\"\n",
+    "    path = Path(path).expanduser()\n",
+    "\n",
+    "    with path.open(\"r\", encoding=\"utf-8\") as f:\n",
+    "        data = json.load(f)\n",
+    "\n",
+    "    if not isinstance(data, list):\n",
+    "        raise ValueError(f\"{path} does not contain a JSON array at the top level.\")\n",
+    "\n",
+    "    # (Optional) sanity-check that every item is a dict\n",
+    "    if not all(isinstance(item, dict) for item in data):\n",
+    "        raise ValueError(\"Not every element in the JSON array is an object.\")\n",
+    "\n",
+    "    return data\n",
+    "\n",
+    "\n",
+    "def extract_description(predict: str) -> Optional[str]:\n",
+    "    \"\"\"\n",
+    "    Extracts the content of the <answer>…</answer> block from `predict`.\n",
+    "    Returns the inner text (with leading/trailing whitespace stripped),\n",
+    "    or None if no <answer> tag is found.\n",
+    "    \"\"\"\n",
+    "    match = re.search(r\"<description>([\\s\\S]*?)</description>\", predict, re.DOTALL)\n",
+    "    if not match:\n",
+    "        return None\n",
+    "    return match.group(1).strip()\n",
+    "\n",
+    "\n",
+    "def accuracy_reward(predict: str, ground_truth: str) -> float:\n",
+    "    answer = extract_boxed_content(predict)\n",
+    "    # answer = extract_answer(predict)\n",
+    "    return 1.0 if grade_answer(answer, ground_truth) else 0.0"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "9fb984e7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def load_json_dir(root: str | Path, *, verbose: bool = True) -> Dict[str, List[Any]]:\n",
+    "    \"\"\"\n",
+    "    Traverse *root* recursively and return {file_stem: parsed_json_data}.\n",
+    "\n",
+    "    • Files that are empty or contain invalid JSON are skipped with a warning.\n",
+    "      Set verbose=False to silence the warnings.\n",
+    "    \"\"\"\n",
+    "    root = Path(root).expanduser().resolve()\n",
+    "    out: Dict[str, List[Any]] = {}\n",
+    "\n",
+    "    for path in root.rglob(\"*.json\"):\n",
+    "        try:\n",
+    "            with path.open(\"r\", encoding=\"utf-8\") as f:\n",
+    "                data = json.load(f)\n",
+    "            out[path.stem] = data\n",
+    "        except json.JSONDecodeError as err:\n",
+    "            if verbose:\n",
+    "                print(f\"[skip] {path} – invalid JSON ({err})\")\n",
+    "        except Exception as err:\n",
+    "            if verbose:\n",
+    "                print(f\"[skip] {path} – {err}\")\n",
+    "\n",
+    "    return out"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "c8e29fcb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# folder_dir = './gemini-flash'\n",
+    "folder_dir = './gemini-pro'\n",
+    "# folder_dir = './gemini-pro-pro'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "fad0547b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "dict_keys(['realWorldQA', 'clevr_count_70k', 'mmmu-pro', 'mathvision', 'mmstar', 'mmmu-pro-vision', 'mm-vet', 'mmmu_pro_10options', 'mathvista', 'visnumbench'])"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "datas = load_json_dir(folder_dir)\n",
+    "\n",
+    "datas.keys()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "e74dd8dd",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "realWorldQA: 0.6862745098039216\n",
+      "clevr_count_70k: 0.7108571428571429\n",
+      "mmmu-pro: 0.6105527638190955\n",
+      "mathvision: 0.36875\n",
+      "mmstar: 0.6633333333333333\n",
+      "mmmu-pro-vision: 0.5256410256410257\n",
+      "mm-vet: 0.3302752293577982\n",
+      "mmmu_pro_10options: 0.49243379571248425\n",
+      "mathvista: 0.554\n",
+      "visnumbench: 0.28835978835978837\n"
+     ]
+    }
+   ],
+   "source": [
+    "indices = {}\n",
+    "\n",
+    "for file, answers in datas.items():\n",
+    "    indices[file]=[]\n",
+    "    acc = 0\n",
+    "    for index, ele in enumerate(answers):\n",
+    "        solution = ele['solution']\n",
+    "        prediction = ele['predictions'][0]\n",
+    "        accuracy = accuracy_reward(prediction, solution)\n",
+    "        acc += accuracy\n",
+    "        \n",
+    "        if accuracy == 1:\n",
+    "            indices[file].append(index)\n",
+    "        \n",
+    "    print(f'{file}: {acc/len(answers)}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "99761358",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "KeyError",
+     "evalue": "'MLLM_rlvr_train'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
+      "\u001b[31mKeyError\u001b[39m                                  Traceback (most recent call last)",
+      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[7]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m \u001b[38;5;28mlen\u001b[39m(\u001b[43mdatas\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mMLLM_rlvr_train\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m]\u001b[49m)\n",
+      "\u001b[31mKeyError\u001b[39m: 'MLLM_rlvr_train'"
+     ]
+    }
+   ],
+   "source": [
+    "len(datas['MLLM_rlvr_train'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "cb380a0c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "dict_keys(['realWorldQA', 'MLLM_hotpot_train', 'mmmu-pro', 'mmstar', 'mm-vet', 'mathvista'])"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "indices.keys()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9367bc67",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "realWorldQA: 0.6972477064220184\n",
+    "mmmu-pro: 0.5646606914212549\n",
+    "mmstar: 0.6061433447098976\n",
+    "mm-vet: 0.6018518518518519\n",
+    "mathvista: 0.5822401614530777"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "08286602",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d033bd06",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8f7a73e5",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "84f260ed",
+   "metadata": {},
+   "source": [
+    "# Construct indices to merge datasets"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "6c771d63",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "description_folder_dir = './gpt_o1_outputs'\n",
+    "description_outputs = load_json_dir(description_folder_dir)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "d8e03bf5",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "891\n",
+      "2694\n"
+     ]
+    }
+   ],
+   "source": [
+    "file = 'mathvision'\n",
+    "print(len(description_outputs[file]))\n",
+    "print(len(datas[file]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "6a814cba",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# idx = 1200\n",
+    "# print(description_outputs[file][idx])\n",
+    "# print('-'*10)\n",
+    "# print(datas[file][idx])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "7c08cff1",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'index': 0,\n",
+       " 'problem': 'Are there any states with a number of reporters between 376-385? Options:\\nA. No\\nB. Yes',\n",
+       " 'solution': 'B',\n",
+       " 'predictions': ['<think> The task is to determine if there are any states with a number of reporters between 376-385. The map shows states in two different colors corresponding to two ranges of reporters: 373-375 and 376-385. I need to identify states in the color representing the range 376-385.\\n\\nThe legend indicates two colors:\\n- A lighter color for the range 373-375.\\n- A darker color for the range 376-385.\\n\\nI will look at the map to identify states shaded with the darker color. The states shaded in the darker color are:\\n- California\\n- North Dakota\\n- South Dakota\\n- Iowa\\n- Missouri\\n- Illinois\\n- Kentucky\\n- West Virginia\\n- New Jersey\\n- Massachusetts\\n\\nThese states are not in the lighter color range, hence they must have a number of reporters between 376-385. Therefore, there are indeed states with a number of reporters in the range 376-385.\\n\\nThe correct option is B. Yes, because there are several states shaded in the darker color that indicates the range 376-385. </think> \\\\boxed{B}']}"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "datas['MLLM_hotpot_train'][0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "c3e8619a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "MLLM_hotpot_train: 0.2949054259284827; dataset len: 14486\n",
+      "mathverse: 0.18071065989847715; dataset len: 3940\n"
+     ]
+    }
+   ],
+   "source": [
+    "indices = {}\n",
+    "\n",
+    "for file, answers in datas.items():\n",
+    "    # try:\n",
+    "        indices[file]=[]\n",
+    "        # description_data = description_outputs[file]\n",
+    "        acc = 0\n",
+    "        for i, ele in enumerate(answers):\n",
+    "            solution = ele['solution']\n",
+    "            prediction = ele['predictions'][0]\n",
+    "            datas_index = ele['index']\n",
+    "            \n",
+    "            # print(description)\n",
+    "            # break\n",
+    "            accuracy = accuracy_reward(prediction, solution)\n",
+    "            # acc += accuracy\n",
+    "            \n",
+    "            if accuracy == 1:\n",
+    "                # if description is not None:\n",
+    "                indices[file].append(datas_index)\n",
+    "                acc += accuracy\n",
+    "            \n",
+    "        print(f'{file}: {acc/len(answers)}; dataset len: {len(answers)}')\n",
+    "    # except Exception as e:\n",
+    "    #     print(f\"Exception caught: {e} for file: {file}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "ca869a96",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Exception caught: name 'description_outputs' is not defined for file: MLLM_hotpot_train\n",
+      "Exception caught: name 'description_outputs' is not defined for file: mathverse\n"
+     ]
+    }
+   ],
+   "source": [
+    "indices = {}\n",
+    "texts = {}\n",
+    "for file, answers in datas.items():\n",
+    "    try:\n",
+    "        indices[file]=[]\n",
+    "        texts[file] = []\n",
+    "        description_data = description_outputs[file]\n",
+    "        # ---------- 1) make a hash‑map: index  ->  description item ----------\n",
+    "        desc_by_idx = {item[\"index\"]: item for item in description_data}\n",
+    "        \n",
+    "        acc = 0\n",
+    "        for i, ele in enumerate(answers):\n",
+    "            solution = ele['solution']\n",
+    "            prediction = ele['predictions'][0]\n",
+    "            data_idx   = ele[\"index\"]          # the index in the answers item\n",
+    "            \n",
+    "            try:\n",
+    "                desc_item  = desc_by_idx.get(data_idx)\n",
+    "                extracted_description = extract_description(desc_item['predictions'][0])\n",
+    "            except:\n",
+    "                extracted_description = None\n",
+    "\n",
+    "            # print(description)\n",
+    "            # break\n",
+    "            accuracy = accuracy_reward(prediction, solution)\n",
+    "            # acc += accuracy       \n",
+    "   \n",
+    "            # print('data: ', description_data)\n",
+    "            # print('-'*10)\n",
+    "            # print('data1: ', ele)\n",
+    "            # break\n",
+    "            \n",
+    "            \n",
+    "            if accuracy == 1:\n",
+    "                if extracted_description is not None:\n",
+    "                    indices[file].append(data_idx)\n",
+    "                    curr_text = '<description>\\n' + extracted_description + '/n</description>' + prediction\n",
+    "                    texts[file].append(curr_text)    \n",
+    "                \n",
+    "                acc += accuracy\n",
+    "            \n",
+    "        print(f'{file}: {acc/len(answers)}; dataset len: {len(answers)}')\n",
+    "    except Exception as e:\n",
+    "        print(f\"Exception caught: {e} for file: {file}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "2d3594e0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "indices_by_dataset = indices"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "4b0a1872",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "K: realWorldQA; V len: 514\n",
+      "K: MLLM_hotpot_train; V len: 0\n",
+      "K: mmmu-pro; V len: 389\n",
+      "K: mathvision; V len: 328\n",
+      "K: mmstar; V len: 512\n",
+      "K: mm-vet; V len: 65\n",
+      "K: mathvista; V len: 457\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "2265"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "total = 0\n",
+    "for k, v in indices_by_dataset.items():\n",
+    "    print(f'K: {k}; V len: {len(v)}')\n",
+    "    total += len(v)\n",
+    "    \n",
+    "total"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4dba6e3c",
+   "metadata": {},
+   "source": [
+    "### Add it for MLLM hotpot train"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "5d453890",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[skip] /apdcephfs_cq11/share_1603164/user/zongxia/workspace/C-gemini-answers/gemini-flash/clevr_count_70k.json – invalid JSON (Expecting value: line 1 column 1 (char 0))\n",
+      "14486\n",
+      "MLLM_hotpot_train: 0.2949054259284827; dataset len: 14486\n",
+      "3940\n",
+      "mathverse: 0.18071065989847715; dataset len: 3940\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "4272"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "indices = {}\n",
+    "\n",
+    "hotpot_description_folder_dir = './gemini-flash'\n",
+    "hotpot_description_outs = load_json_dir(hotpot_description_folder_dir)\n",
+    "\n",
+    "for file, answers in hotpot_description_outs.items():\n",
+    "        try:\n",
+    "            print(len(answers))\n",
+    "            indices[file]=[]\n",
+    "            texts[file] = []\n",
+    "            acc = 0\n",
+    "            for i, ele in enumerate(answers):\n",
+    "                solution = ele['solution']\n",
+    "                prediction = ele['predictions'][0]\n",
+    "                datas_index = ele['index']\n",
+    "                \n",
+    "                # print(description)\n",
+    "                # break\n",
+    "                accuracy = accuracy_reward(prediction, solution)\n",
+    "                # acc += accuracy\n",
+    "                \n",
+    "                if accuracy == 1:\n",
+    "                    indices[file].append(datas_index)\n",
+    "                    texts[file].append(prediction)\n",
+    "                    acc += accuracy\n",
+    "                \n",
+    "            print(f'{file}: {acc/len(answers)}; dataset len: {len(answers)}')\n",
+    "        except Exception as e:\n",
+    "            print(f\"Exception caught: {e} for file: {file}\")\n",
+    "\n",
+    "len(indices['MLLM_hotpot_train'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "8f4fe74e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "len(idxs) = 14486  min = 0  max = 14485\n",
+      "missing count : 0\n",
+      "first 20 gaps : []\n"
+     ]
+    }
+   ],
+   "source": [
+    "idxs = [ele['index'] for ele in hotpot_description_outs['MLLM_hotpot_train']]\n",
+    "\n",
+    "\n",
+    "print(\"len(idxs) =\", len(idxs), \" min =\", min(idxs), \" max =\", max(idxs))\n",
+    "# → len(idxs) == 6105, min == 0 (maybe), max == 6463\n",
+    "\n",
+    "# 2) find every number that *should* be there but isn’t\n",
+    "expected = set(range(min(idxs), max(idxs) + 1))   # full consecutive range\n",
+    "missing  = sorted(expected - set(idxs))\n",
+    "\n",
+    "print(\"missing count :\", len(missing))\n",
+    "print(\"first 20 gaps :\", missing[:20])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "411dcfc7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "indices_by_dataset = indices"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "ce4cea20",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "dict_keys(['MLLM_hotpot_train', 'mathverse'])\n",
+      "dict_keys(['MLLM_hotpot_train', 'mathverse'])\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(indices_by_dataset.keys())\n",
+    "print(texts.keys())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "2a3ea275",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "4272"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(indices_by_dataset['MLLM_hotpot_train'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "08197397",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[14471, 14473, 14474, 14476, 14477, 14478, 14480, 14481, 14484, 14485]"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "indices_by_dataset['MLLM_hotpot_train'][-10:]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "bd2b91ff",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/usr/local/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "filename: zli12321/MLLM_hotpot_train\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Flattening the indices: 100%|██████████| 4272/4272 [00:03<00:00, 1282.44 examples/s]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "filename: zli12321/mathverse\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Generating test split: 3940 examples [00:00, 13229.68 examples/s]\n",
+      "Flattening the indices: 100%|██████████| 712/712 [00:00<00:00, 48814.82 examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Dataset({\n",
+      "    features: ['problem', 'answer', 'images', 'outputs'],\n",
+      "    num_rows: 4984\n",
+      "})\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "from datasets import load_dataset, concatenate_datasets\n",
+    "\n",
+    "BASE_REPO   = \"zli12321/\"          # prefix for every dataset id\n",
+    "kept_splits = []\n",
+    "\n",
+    "for short_name, keep in indices_by_dataset.items():\n",
+    "    try:\n",
+    "        if not keep:                              # nothing to keep → skip\n",
+    "            continue\n",
+    "\n",
+    "        # -----------------------------------------------------------------\n",
+    "        # 1) ensure `keep` and its matching texts are sorted *together*\n",
+    "        # -----------------------------------------------------------------\n",
+    "        idxs   = keep\n",
+    "        outs   = texts[short_name]\n",
+    "\n",
+    "        # idxs and outs were built in parallel, so they are aligned.\n",
+    "        # If you want the rows in ascending order, sort both lists together:\n",
+    "        order  = sorted(range(len(idxs)), key=idxs.__getitem__)\n",
+    "        idxs   = [idxs[i]  for i in order]        # sorted indices\n",
+    "        outs   = [outs[i]  for i in order]        # matching outputs\n",
+    "\n",
+    "        # -----------------------------------------------------------------\n",
+    "        # 2) load, slice, and keep only the three original columns\n",
+    "        # -----------------------------------------------------------------\n",
+    "        full_name = f\"{BASE_REPO}{short_name}\"\n",
+    "        \n",
+    "        print(f'filename: {full_name}')\n",
+    "        split     = \"train\" if \"MLLM_hotpot_train\" in short_name else \"test\"\n",
+    "\n",
+    "        ds = load_dataset(full_name, split=split, trust_remote_code=True)\n",
+    "        ds = ds.select(idxs)                      # keep only those rows\n",
+    "        \n",
+    "        # print(f'filename: {full_name}; len: {len(ds)}')\n",
+    "\n",
+    "        cols_to_keep = {\"problem\", \"images\", \"answer\"}\n",
+    "        ds = ds.remove_columns([c for c in ds.column_names if c not in cols_to_keep])\n",
+    "\n",
+    "        # -----------------------------------------------------------------\n",
+    "        # 3) add the new column\n",
+    "        # -----------------------------------------------------------------\n",
+    "        ds = ds.add_column(\"outputs\", outs)       # len(outs) == len(ds)\n",
+    "\n",
+    "        kept_splits.append(ds)\n",
+    "    except Exception as e:\n",
+    "        print(f\"dataset len: {len(ds)}\")\n",
+    "        print(f'{short_name} Failed: {e}')\n",
+    "\n",
+    "# ---------------------------------------------------------------------\n",
+    "# 4) concatenate everything into one big dataset\n",
+    "# ---------------------------------------------------------------------\n",
+    "combined = concatenate_datasets(kept_splits)\n",
+    "\n",
+    "print(combined)                # verify\n",
+    "# combined.save_to_disk(\"combined.arrow\")     # or .to_parquet(...)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "id": "cb8bfe20",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Creating parquet from Arrow format: 100%|██████████| 39/39 [00:17<00:00,  2.18ba/s]\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "909006342"
+      ]
+     },
+     "execution_count": 29,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "combined.to_parquet(\"./hf_upload_train/train.parquet\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "5b7aed77",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def save_any_image(img_obj, out_base: Path) -> Path:\n",
+    "    \"\"\"\n",
+    "    Save *img_obj* (str | dict | PIL.Image) to disk.\n",
+    "    Returns the *Path* actually written (possibly .png if alpha).\n",
+    "    \"\"\"\n",
+    "    import io, shutil\n",
+    "    from PIL import Image\n",
+    "\n",
+    "    # 1) resolve a PIL.Image ---------------------------------------------------\n",
+    "    if isinstance(img_obj, str):                 # already a path\n",
+    "        pil = Image.open(img_obj)\n",
+    "\n",
+    "    elif isinstance(img_obj, dict):              # HF Image feature\n",
+    "        if img_obj.get(\"path\"):\n",
+    "            pil = Image.open(img_obj[\"path\"])\n",
+    "        else:\n",
+    "            pil = Image.open(io.BytesIO(img_obj[\"bytes\"]))\n",
+    "\n",
+    "    else:                                        # PIL.Image.Image\n",
+    "        pil = img_obj\n",
+    "\n",
+    "    # 2) choose format & filename ---------------------------------------------\n",
+    "    suffix   = \".jpg\"\n",
+    "    img_mode = pil.mode\n",
+    "\n",
+    "    if img_mode in (\"RGBA\", \"LA\", \"P\"):\n",
+    "        # keep alpha by switching to PNG (or call .convert(\"RGB\") to stay JPEG)\n",
+    "        suffix = \".png\"\n",
+    "\n",
+    "    out_path = out_base.with_suffix(suffix)\n",
+    "\n",
+    "    # 3) convert if you insist on JPG without alpha\n",
+    "    if suffix == \".jpg\" and img_mode != \"RGB\":\n",
+    "        pil = pil.convert(\"RGB\")\n",
+    "\n",
+    "    # 4) write -----------------------------------------------------------------\n",
+    "    pil.save(out_path)\n",
+    "    return out_path\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "358edaa6",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "writing images: 100%|██████████| 4984/4984 [14:38<00:00,  5.67it/s]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅  Done: 4984 items saved.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os, io, json, shutil\n",
+    "from pathlib import Path\n",
+    "from PIL import Image\n",
+    "from tqdm import tqdm          # optional progress bar\n",
+    "\n",
+    "# ------------------------------------------------------------------ #\n",
+    "# directory setup\n",
+    "# ------------------------------------------------------------------ #\n",
+    "OUT_DIR = Path(\"sft_description\")\n",
+    "OUT_DIR.mkdir(exist_ok=True)   # creates folder if missing\n",
+    "\n",
+    "json_records = []\n",
+    "\n",
+    "# ------------------------------------------------------------------ #\n",
+    "# main loop\n",
+    "# ------------------------------------------------------------------ #\n",
+    "for idx, row in enumerate(tqdm(combined, desc=\"writing images\")):\n",
+    "    img_path = save_any_image(row[\"images\"], OUT_DIR / str(idx))\n",
+    "    json_records.append({\n",
+    "        \"messages\": [\n",
+    "            {\"content\": row[\"problem\"],  \"role\": \"user\"},\n",
+    "            {\"content\": row[\"outputs\"],  \"role\": \"assistant\"}\n",
+    "        ],\n",
+    "        \"images\": [str(img_path)]\n",
+    "    })\n",
+    "\n",
+    "# ------------------------------------------------------------------ #\n",
+    "# write the JSONL / JSON\n",
+    "# ------------------------------------------------------------------ #\n",
+    "with open(\"sft_description.json\", \"w\", encoding=\"utf-8\") as f:\n",
+    "    json.dump(json_records, f, ensure_ascii=False, indent=2)\n",
+    "\n",
+    "print(f\"✅  Done: {len(json_records)} items saved.\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d4e56b70",
+   "metadata": {},
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "adc502bc",
+   "metadata": {},
+   "source": [
+    "### Now process the data for Hotpot Train"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e84f2aa2",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "54356d4e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "from openai import OpenAI\n",
+    "from concurrent.futures import ThreadPoolExecutor, as_completed\n",
+    "from time import sleep\n",
+    "from typing import List, Dict, Any, Optional\n",
+    "from openai import OpenAI\n",
+    "from __future__ import annotations\n",
+    "import json\n",
+    "from pathlib import Path\n",
+    "from typing import Any, Dict, Iterable, List, Union"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "5caaaa06",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'index': 0,\n",
+       " 'problem': 'Are there any states with a number of reporters between 376-385? Options:\\nA. No\\nB. Yes',\n",
+       " 'solution': 'B',\n",
+       " 'predictions': ['<description>The image is a map of the United States, with each state colored according to the number of reporters in that state. The title of the map is \"The Number of reporters in the USA\". There is a legend in the bottom right corner. States colored in a light beige color have between 373-375 reporters. States colored in a dark purple color have between 376-385 reporters. Several states are colored dark purple, including Washington, Montana, North Dakota, South Dakota, Iowa, Missouri, Louisiana, Utah, Nevada, California, Virginia, Maryland, and New Hampshire. Alaska and Hawaii are also shown. </description>\\n<think>The question asks if there are any states with a number of reporters between 376-385. The legend indicates that states with 376-385 reporters are colored dark purple. The map shows several states colored dark purple. Therefore, the answer is yes. </think>\\n\\\\boxed{Yes}']}"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "data[0]"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

evaluate_single.py

@@ -0,0 +1,77 @@
+from utils.gemini_eval import *
+from mathruler.grader import extract_boxed_content
+import json
+from typing import List, Dict, Union, Any
+from pathlib import Path
+from tqdm import tqdm
+import logging
+logging.getLogger().setLevel(logging.ERROR)
+import json
+from pathlib import Path
+from tqdm import tqdm
+import concurrent.futures
+from datasets import load_dataset
+
+
+def dump_json(
+    data: List[Dict[str, Any]],
+    path: Union[str, Path],
+    *,
+    indent: int | None = 2,
+    ensure_ascii: bool = False
+) -> None:
+    """
+    Save `data` (a list of dictionaries) to `path` in JSON format.
+
+    Parameters
+    ----------
+    data : list[dict]
+        The objects you want to serialize.
+    path : str | pathlib.Path
+        Where to write the JSON file.  Parent directories are created if needed.
+    indent : int | None, default=2
+        How many spaces to use for pretty-printing.  Set to `None` for a single-line file.
+    ensure_ascii : bool, default=False
+        If False, non-ASCII characters are written as UTF-8; if True, they're escaped.
+
+    Raises
+    ------
+    TypeError
+        If `data` contains objects that `json` cannot serialize.
+    OSError
+        If the file cannot be created or written.
+    """
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+    with path.open("w", encoding="utf-8") as f:
+        json.dump(data, f, indent=indent, ensure_ascii=ensure_ascii)
+        f.write("\n")  # final newline (POSIX style)
+
+
+
+
+ONLY_FILE = './gemini-pro/visnumbench.json'
+output_file = './eval_out/gemini-pro/visnumbench.json'
+
+
+ONLY_FILE   = Path(ONLY_FILE)
+with ONLY_FILE.open("r", encoding="utf-8") as f:
+    data = json.load(f) 
+    
+outputs = []
+
+for ele in tqdm(data):
+    problem = ele['problem'].replace('<image>', '')
+    reference = ele['solution']
+    candidate = extract_boxed_content(ele['predictions'][0])
+    
+    judgment = generate(problem, reference, candidate)
+    # print(judgment)
+    ele['judgment'] = judgment
+    outputs.append(ele)
+    
+    
+
+
+dump_json(outputs, output_file)

gemini_key.json

@@ -0,0 +1,13 @@
+{
+  "type": "service_account",
+  "project_id": "tencent-gemini-omd01",
+  "private_key_id": "c2eab17392a5b8ceb2d389b439573ad26a33700e",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCpxIaQ1HXmZT4a\nIiyi8EZxAFuPq8dog4+6cDs7HBmXXkqn7oUMWbEneLlCdC7TAAckRCqljIpjQiRE\nC6bqtSCjc5GEOOCknKMBfcwu/I4Pc5Tmhi8eu4alDocJgtnRMnzZwHy/Ama/FJ5d\nKnR24DubKJQUf9mCfvxgJj9xNw7c57EMJ2X/o7BjecHOiKZX1fN5J97yviE+wIm/\nxubkYyMuEM+HLhQWo6fI7SV1/qe3i94OGnYmSnLD6G7IN0kjoZj3n88+JGJpxJKp\nQ3Lv/gRQPjGhOylYNqsvulirKdeKoXWhMmV9Z71FjshMMedWoAnH7ijdh4SJALgY\nD1h9QbYpAgMBAAECggEAB9TMJ6fYAndJd0Impfjd1FOhuvhqcUdJsjOAKqNgCNat\ngfP4849lon5T/sfw6wSAJSuh01OEL2el2EpjT5toXzHdv2gyRgBVwqQExGbGai8O\nP4NmxAz+0o601aWmxPAxHd2zw3SUZxxyi7YBe68Eq40ie5REOoaqFLNe+irXMYMe\nkpellvArBamyYjjGYhDK4CRcHIotc3ZdfcJR5wqjViUnGS1onhdBSMxovpYnXNj9\nHK+pVlaKzvJQ5ovnr7Ic9XgDHdo23s/1zk8/Hh0wLEGzOzz1L+bMdMeZ8JR8CVQ2\nZ2+DM7LILpm3yUDX/LtteCTPAjG1prNv1jrXYExyEwKBgQDo2DeEMlV3zw0BqGQy\nHsMnJngWY6Z9tVCnLC5fEAEV9uDbKkt8OIenCwrZlsnO1XeFyig4UqgOuQfHQT0v\nE0FtoYfnjYzY0pogXzW1jC/flvSMLxz+KzI35zlRNOlSTTVafy4EsNN0kayTgHzO\nI8Xh0VBYkddypGxG/WwiF9adSwKBgQC6pn1drkbmTMDcjT2CnipxWsgmTr1fD8vo\nRIypaq+AU89ULYjdUaMTv2uHpE5wVQe+va2QhCUJ6pNwkV3c2KjTQ0TOXjaTJJSn\nFAJ4MLtVneBcz2eJHIGP5fJIjQmGgwWU8h+0LShQUdVUmGUR8763jzpClZvJxIkM\nmot6qgYV2wKBgEuOLaWV96ni4+OP0sN4u/auQvVw7IuKFFvKuFlchh2seJZliQ0M\nAuivapvklCOrnRcq3BY6rBHq9J0xjDsEUozSh5kZk4SgAidS/cilbrts7nm2p//J\n4IfHXg/9zWBJcXCmKDaZcmQ3CPrsDJOPhBycoSe8W171/7Shcz804Q01AoGAYKrV\nhu05vxDFWfS0hK+R379apaxmG5O80YCfMScV2eqOGFS065raUOH0uP15umfvaPQn\nrg8id65LyiMfb7+uQCw4uIDG1xI9AwMz/DeQ7lij2K16O+LNn09CWhzhcA7vlyKH\nFPPGS0L2r6d1wQRyI/NEAzQkySzGpgZsco0YNb0CgYEAjETeHxD/itWyHOg6y49c\nw74Yiwk1eFMg7C3lNXd5j8tRk1+2QDhcSMJo6QdRomzyDL9DIxCWUJblVi6hJof/\nE4PH07JBaRAEnWMHjJpBzXriQ5VhWU0MRwpaHOZ2ZAoOkkiqKZIhwryBvfoZa07Q\noArMcZ4xuOoFhE9pOE6g5nE=\n-----END PRIVATE KEY-----\n",
+  "client_email": "[email protected]",
+  "client_id": "104935904768461518534",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/ailab01%40tencent-gemini-omd01.iam.gserviceaccount.com",
+  "universe_domain": "googleapis.com"
+}

generate_answer.py

@@ -0,0 +1,324 @@
+import json, os
+from pathlib import Path
+from typing import List
+from datasets import load_dataset
+from PIL import Image
+from tqdm import tqdm
+import concurrent.futures as cf
+import os
+from openai import AzureOpenAI
+from typing import Set, List, Dict, Any
+import time
+import pandas as pd
+from tqdm import tqdm
+import io
+import base64
+import imghdr
+from io import BytesIO
+from mimetypes import guess_type
+import base64
+import time
+from datasets import load_dataset, Features, Sequence, Value, Image as HFImage, ClassLabel
+from PIL import Image
+# from azure.core.exceptions import AzureError
+import concurrent.futures as cf
+import os
+from typing import List
+import os
+from io import BytesIO
+
+import vertexai
+from vertexai import generative_models
+from vertexai.generative_models import GenerativeModel, Part
+from datasets import load_dataset
+from PIL import Image as PILImage
+
+
+# 0) Point at your service account
+os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = './gemini_key.json'
+
+# 1) Your generation & safety configs (unchanged)
+generation_config = {
+    "max_output_tokens": 2048,
+    "temperature": 0.4,
+    "top_p": 0.4,
+    "top_k": 32,
+}
+safety_settings = {
+    generative_models.HarmCategory.HARM_CATEGORY_HATE_SPEECH:
+        generative_models.HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE,
+    generative_models.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT:
+        generative_models.HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE,
+    generative_models.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT:
+        generative_models.HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE,
+    generative_models.HarmCategory.HARM_CATEGORY_HARASSMENT:
+        generative_models.HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE,
+}
+
+
+TIMEOUT_CODES = {408, 504, 524}
+
+
+DATASETS = [
+    # "zli12321/realWorldQA",
+    # "zli12321/mmmu-pro",
+    # "zli12321/mathvista",
+    # "zli12321/mm-vet",
+    # "zli12321/mmstar",
+    # "zli12321/mathvision",
+    # "zli12321/MLLM_hotpot_train",
+    "zli12321/MLLM_test"
+    # "BUAADreamer/clevr_count_70k",
+    # "zli12321/mathverse"
+    # "zli12321/MLLM_rlvr_train"
+    # "zli12321/mmmu-pro-vision",
+    # "zli12321/visnumbench",
+    # "zli12321/mmmu_pro_10options"
+]
+
+# ---------------------------------------------------------------------
+# 1) CONFIG – adjust as you like
+# ---------------------------------------------------------------------
+# DATA_OUT = "./gpt_outputs/realworldQA.json"
+N_GEN     = 1              # ⇐ how many completions per prompt
+retry_delay = 10
+
+# QUESTION_TEMPLATE = (
+#     "You are tasked with analyzing an image to generate a detailed description to help you answer the question. First analyze the image and produce a self-contained description—detailed enough that can lead to the correct answer. Wrap the entire description in <description> </description> tags.\n Next, engage in an internal dialogue and include self-reflection or verification in your reasoning process. Provide your detailed, step-by-step reasoning based on the image description information and image, and enclose this part within <think> </think> tags.\n Finally, provide a single word or phrase answer to the question in \\boxed{}.\nThe output format should be: <description> image description here </description> <think> reasoning process here </think> \\boxed{FINAL ANSWER here}. Please only include the single letter choice as your answer for multiple choice questions."
+#     "Question: {Question}\n"
+# )
+
+
+QUESTION_TEMPLATE = (
+    "You are tasked with analyzing an image and answer a question. First engage in an internal dialogue and include self-reflection or verification in your reasoning process. Provide your detailed, step-by-step reasoning based on the image description information and image, and enclose this part within <think> </think> tags.\n Finally, provide a single word or phrase answer to the question in \\boxed{}.\nThe output format should be: <think> reasoning process here </think> \\boxed{FINAL ANSWER here}."
+    "Question: {Question}\n"
+)
+
+
+# QUESTION_TEMPLATE = (
+#     "You are tasked with analyzing an image to generate a detailed description to help you answer the question. Analyze the image and produce a self-contained description—detailed enough that can lead to the correct answer. Wrap the entire description in <description> </description> tags. Then provide a single word or phrase answer to the question in \\boxed{}. The output format should be: <description> image description here </description> \\boxed{FINAL ANSWER here}."
+#     "Question: {Question}\n"
+# )
+
+def is_timeout(err):
+    """Return True if the error (or its cause) is a network timeout."""
+    return isinstance(err, TimeoutError) or isinstance(
+        getattr(err, "__cause__", None), TimeoutError
+    )
+
+
+vertexai.init(project="tencent-gemini-omd01", location="us-central1")
+
+'''Below is for rlvr'''
+# model = GenerativeModel("gemini-2.0-flash")
+
+'''Below is for counting'''
+# model = GenerativeModel("gemini-2.0-flash-lite")
+
+
+'''Below is for Gemini pro pro Accuracy'''
+model = GenerativeModel("gemini-2.5-pro")
+# 3) Load CLEVR‑count 70k and pull the first example’s images list
+
+
+def generate(pil_img, query):
+    # 1) Ensure RGB & re‑encode as PNG in‑memory
+    buf = BytesIO()
+    pil_img.convert("RGB").save(buf, format="PNG")
+    png_bytes = buf.getvalue()
+
+    # 2) Wrap in a Part
+    image_part = Part.from_data(
+        data=png_bytes,
+        mime_type="image/png"
+    )
+    for i in range(2):
+        # 3) Generate
+        try:
+            responses = model.generate_content(
+                contents=[image_part, query],
+                generation_config=generation_config,
+                safety_settings=safety_settings,
+                stream=True,
+            )
+
+            # 4) Collect and return
+            full = ""
+            for chunk in responses:
+                full += chunk.text
+        except Exception as e:
+            full = "No Text"
+            print(f'Failed generating: {e}')
+            time.sleep(5)
+        
+    
+    return full
+
+
+
+# ---------------------------------------------------------------------
+# 2) YOUR MODEL / API CALL – plug in here
+# ---------------------------------------------------------------------
+def generate_answer(image, messages) -> str:
+    """
+    Replace the body of this function with whatever you use to talk to
+    your model (e.g. OpenAI, Ollama, local HF pipeline, etc.).
+    Must return a *single* string completion.
+    """
+    # raise NotImplementedError(
+    #     "Implement generate_answer(img, prompt_text) to call your model."
+    # )
+    # return azure_gpt4(messages)
+    return generate(image, messages)
+    
+    
+
+# ---------------------------------------------------------------------
+# 3) DATASET & UTILS
+# ---------------------------------------------------------------------
+
+def build_prompt(item) -> str:
+    """Fill QUESTION_TEMPLATE with the current question."""
+    return QUESTION_TEMPLATE.replace("{Question}", item["problem"])
+
+def to_rgb(img: Image.Image) -> Image.Image:
+    return img if img.mode == "RGB" else img.convert("RGB")
+
+def _load_partial(out_path: Path) -> List[Dict[str, Any]]:
+    if not out_path.exists():
+        return []
+    try:
+        with out_path.open("r", encoding="utf-8") as f:
+            return json.load(f)
+    except Exception as err:
+        print(f"[warn] {out_path} could not be read ({err}) – ignoring.")
+        return []
+
+
+def run_dataset(dataset_id: str, n_gen: int = 1) -> None:
+    """Run the generation loop for one dataset, resuming if output exists."""
+    print(f"\n=== Processing {dataset_id} ===")
+
+    # ---- prepare output path ----------------------------------------
+    slug      = dataset_id.split("/")[-1]
+    # DATA_OUT  = Path(f"./gemini-flash/{slug}.json")
+    # DATA_OUT  = Path(f"./gemini-pro/{slug}.json")
+    # DATA_OUT  = Path(f"./gemini-pro-pro/{slug}.json")
+    DATA_OUT  = Path(f"./gemini-cot/{slug}.json")
+    
+
+    DATA_OUT.parent.mkdir(parents=True, exist_ok=True)
+
+    # ---- load existing results (if any) -----------------------------
+    results: List[Dict[str, Any]] = _load_partial(DATA_OUT)
+    done_idx: Set[int]            = {rec["index"] for rec in results}
+    print(f"[{slug}] found {len(done_idx)} previously processed items")
+
+    
+    # ---- load split -------------------------------------------------
+    # if 'count' in dataset_id or 'hotpot' in dataset_id:
+    #     ds = load_dataset(dataset_id, split="train", trust_remote_code=True)
+    # else:
+    #     ds = load_dataset(dataset_id, split="test", trust_remote_code=True)
+    
+    if 'count' in dataset_id or 'hotpot' in dataset_id:
+        ds = load_dataset(dataset_id, split="train")
+    else:
+        try:
+            ds = load_dataset(dataset_id, split="train")
+        except:
+            ds = load_dataset(dataset_id, split="test", trust_remote_code=True)
+        
+
+    # ---- decode images once ----------------------------------------
+    df = ds.to_pandas()
+    try:
+        df["pil_images"] = df["images"].apply(
+            lambda lst: [Image.open(io.BytesIO(d["bytes"])).convert("RGB") for d in lst]
+        )
+        images = [imgs[0] for imgs in df["pil_images"]]
+    except Exception:
+        df["pil_images"] = df["images"].apply(
+            lambda d: Image.open(io.BytesIO(d["bytes"])).convert("RGB")
+        )
+        images = list(df["pil_images"])
+
+    # ---- main generation loop --------------------------------------
+    with cf.ThreadPoolExecutor(max_workers=n_gen) as pool:      # <-- here
+        for idx, item in enumerate(
+            tqdm(ds, desc=f"generating · {slug}",
+                 initial=len(done_idx), total=len(ds))
+        ):
+            if idx in done_idx:
+                continue
+
+            prompt_txt = build_prompt(item)
+            # image_url  = pil_image_to_data_url(images[idx])
+            # messages   = [{"instruction": prompt_txt, "image": image_url}]
+
+            # launch `n_gen` concurrent calls
+            futures = [pool.submit(generate_answer, images[idx], prompt_txt)
+                       for _ in range(n_gen)]                  # <-- here
+            answers = [f.result() for f in futures if f.result()]
+
+            if answers:
+                results.append(
+                    dict(
+                        index       = idx,
+                        problem     = item["problem"],
+                        solution    = item["answer"],
+                        predictions = answers,
+                    )
+                )
+            DATA_OUT.write_text(json.dumps(results, indent=2, ensure_ascii=False))
+    print(f"✅ {slug}: finished {len(results)} samples → {DATA_OUT}")
+    
+
+# --------------------------- 2. run_all -------------------------------
+def run_all(
+    datasets: list,                    # list[str]  *or*  list[tuple[str,int]]
+    default_n_gen: int = 1,
+    max_workers: int | None = None,
+) -> None:
+    """
+    Launch `run_dataset` for every entry in *datasets*.
+
+    `datasets` may contain:
+      • "foo/bar"                      -> uses default_n_gen
+      • ("foo/bar", 8)                 -> uses 8 for that file
+    """
+    if max_workers is None:
+        max_workers = min(len(datasets), 32)
+
+    print(f"\nLaunching {len(datasets)} dataset jobs "
+          f"({max_workers} workers)…\n")
+
+    with cf.ThreadPoolExecutor(max_workers=max_workers) as pool:
+        fut_to_name = {}
+        for entry in datasets:
+            if isinstance(entry, tuple):
+                ds_id, n_gen = entry
+            else:
+                ds_id, n_gen = entry, default_n_gen
+            fut = pool.submit(run_dataset, ds_id, n_gen)
+            fut_to_name[fut] = ds_id
+
+        for fut in cf.as_completed(fut_to_name):
+            name = fut_to_name[fut]
+            try:
+                fut.result()
+            except Exception as exc:
+                print(f"❌ {name} failed: {exc!r}")
+            else:
+                print(f"✅ {name} done")
+
+# ---------------------------------------------------------------------
+# ENTRY-POINT
+# ---------------------------------------------------------------------
+if __name__ == "__main__":
+    run_all(DATASETS, max_workers=min(len(DATASETS), os.cpu_count() * 2))
+
+
+
+'''Below is code for gemini inference'''
+

qwen_caption_cot.py

@@ -0,0 +1,190 @@
+from __future__ import annotations
+import pandas as pd
+from openai import OpenAI
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from time import sleep
+from typing import List, Dict, Any, Optional
+from openai import OpenAI
+import json
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Union
+import re
+import datetime
+from typing import Dict, List, Optional
+from mathruler.grader import extract_boxed_content, grade_answer
+import math
+from tqdm.auto import tqdm
+
+
+def extract_description(predict: str) -> Optional[str]:
+    """
+    Extracts the content of the <answer>…</answer> block from `predict`.
+    Returns the inner text (with leading/trailing whitespace stripped),
+    or None if no <answer> tag is found.
+    """
+    match = re.search(r"<description>([\s\S]*?)</description>", predict, re.DOTALL)
+    if not match:
+        return predict
+    return match.group(1).strip()
+
+# curl http://29.81.228.243:8081 /v1/models
+client = OpenAI(
+    base_url="http://29.81.244.54:8081/v1",  # your vLLM server
+    api_key="ANYKEY",                        # if you set --api-key when launching
+)
+
+
+def chat_once(messages):
+    resp = client.chat.completions.create(
+        model="Qwen2.5-VL-72B-Instruct",
+        messages=messages
+    )
+    return resp.choices[0].message.content
+
+
+def chat_batch(
+    client,
+    all_message_batches: List[List[Dict[str, str]]],
+    *,
+    model: str = "Qwen2.5-VL-72B-Instruct",
+    max_workers: int = 8,
+    retries: int = 2,
+    backoff: float = 0.5,
+    timeout: Optional[float] = None,
+) -> List[str]:
+    """
+    Send many chat requests in parallel and return replies as a list of strings,
+    preserving the order of `all_message_batches`.
+    """
+
+    def _chat_once_with_retry(messages: List[Dict[str, str]]) -> str:
+        last_err: Optional[BaseException] = None
+        for attempt in range(retries + 1):
+            try:
+                resp = client.chat.completions.create(
+                    model=model,
+                    messages=messages,
+                    timeout=timeout,
+                )
+                # Different SDKs expose content slightly differently; handle common cases.
+                choice = resp.choices[0]
+                if hasattr(choice, "message") and getattr(choice.message, "content", None) is not None:
+                    return choice.message.content
+                if hasattr(choice, "text") and choice.text is not None:
+                    return choice.text
+                # Fallback to stringifying the choice if structure is unexpected.
+                return str(choice)
+            except Exception as e:
+                last_err = e
+                if attempt < retries:
+                    sleep(backoff * (2 ** attempt))
+        return f"Error: {last_err!r}"
+
+    results: List[Optional[str]] = [None] * len(all_message_batches)
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        future_to_idx = {
+            executor.submit(_chat_once_with_retry, batch): i
+            for i, batch in enumerate(all_message_batches)
+        }
+        for fut in as_completed(future_to_idx):
+            i = future_to_idx[fut]
+            results[i] = fut.result()
+
+    # mypy-friendly cast: no Nones remain at this point
+    return [r if r is not None else "Error: Unknown failure" for r in results]
+
+
+
+def load_json_list(path: Union[str, Path], encoding: str = "utf-8") -> List[Dict[str, Any]]:
+    """
+    Load a JSON file whose top-level structure is a list of dicts.
+
+    Raises:
+        FileNotFoundError, json.JSONDecodeError, TypeError
+    """
+    p = Path(path)
+    with p.open("r", encoding=encoding) as f:
+        data = json.load(f)
+
+    if not isinstance(data, list):
+        raise TypeError(f"Expected top-level JSON to be a list, got {type(data).__name__}")
+
+    for i, item in enumerate(data):
+        if not isinstance(item, dict):
+            raise TypeError(f"Item at index {i} is {type(item).__name__}, expected dict")
+
+    return data
+
+# Prepare a list of different message‐lists you want to send:
+all_message_batches = [
+    [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user",   "content": "Hello, how are you?"}
+    ],
+    [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user",   "content": "Tell me a joke."}
+    ],
+    [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user",   "content": "Tell me a joke."}
+    ],
+    [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user",   "content": "Tell me a joke."}
+    ],
+    # …more batches…
+]
+
+
+res = chat_batch(client, all_message_batches)
+
+prompt_template = '''Text description: {Description}\nQuestion: {Question}\nYou are provided a text description of a problem and a question. Determine the answer to the question based on the text description. First provide an internal step-by-step reasoning within <think> </think> tags, then provide a single word or phrase answer in \\boxed{}.'''
+MODEL = "Qwen2.5-VL-72B-Instruct"
+BATCH_SIZE = 16
+filename = "MLLM_rlvr_train"
+out_file = f'./caption_out/{filename}.json'
+data = load_json_list(f'./gemini-flash/{filename}.json')
+
+
+def to_messages(example: Dict[str, Any]) -> List[Dict[str, str]]:
+    """Use the single string inside `predictions` as the user input."""
+    preds = example.get("predictions")
+    question = example.get("problem")
+    
+    if isinstance(preds, list) and preds:
+        first = preds[0]
+        text = first if isinstance(first, str) else json.dumps(first, ensure_ascii=False)
+        description = extract_description(text)
+        input_question = prompt_template.replace('{Description}', description).replace('{Question}', question)
+    else:
+        input_question = 'None'
+
+    return [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user",   "content": input_question},
+    ]
+
+
+# Ensure output dir exists and start fresh
+Path(out_file).parent.mkdir(parents=True, exist_ok=True)
+with open(out_file, "w", encoding="utf-8"):
+    pass
+
+total = len(data)
+num_batches = math.ceil(total / BATCH_SIZE)
+
+for start in tqdm(range(0, total, BATCH_SIZE),
+                  total=num_batches, desc="Batches", unit="batch"):
+    chunk = data[start : start + BATCH_SIZE]
+    batch_messages = [to_messages(ex) for ex in chunk]
+
+    replies = chat_batch(client, batch_messages, model=MODEL,
+                         max_workers=8, retries=2, backoff=0.5, timeout=None)
+
+    print(replies[0])
+    with open(out_file, "a", encoding="utf-8") as f:
+        for ex, reply in zip(chunk, replies):
+            record = {**ex, "model": MODEL, "model_caption_response": reply}
+            f.write(json.dumps(record, ensure_ascii=False) + "\n")
+        f.flush()