Upload 5 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/tokenlengths.png filter=lfs diff=lfs merge=lfs -text

assets/test.py

@@ -0,0 +1,162 @@
+# download dataset
+# wget https://huggingface.co/datasets/Metin/WikiRAG-TR/resolve/main/data/train.csv
+
+import gzip
+import logging
+import os
+import sys
+from collections import defaultdict
+
+import numpy as np
+import pytrec_eval
+import tqdm
+import pandas as pd
+from sentence_transformers import CrossEncoder, util
+from pylate import evaluation, losses, models, utils, rank
+
+evalResultsDf = None
+question2answer = False
+shortContext = True  # only affects question2context retrieval
+
+if question2answer:
+    document_length = 256
+else:
+    document_length = 8190
+
+model_name_or_paths = [
+    "Y-J-Ju/ModernBERT-base-ColBERT",
+    "99eren99/ColBERT-ModernBERT-base-Turkish-uncased",
+    "jinaai/jina-colbert-v2",
+]
+for model_name_or_path in model_name_or_paths:
+    if "jina" in model_name_or_path:
+        model = models.ColBERT(
+            model_name_or_path=model_name_or_path,
+            query_prefix="[QueryMarker]",
+            document_prefix="[DocumentMarker]",
+            attend_to_expansion_tokens=True,
+            trust_remote_code=True,
+            document_length=document_length,
+        )
+    else:
+        model = models.ColBERT(
+            model_name_or_path=model_name_or_path, document_length=document_length
+        )
+
+    # query length kept default because it pads to query length
+    # and document length set to a high value to avoid truncation, it only truncates the document doesn't pads
+
+    model.eval()
+    model.to("cuda")
+
+    df = pd.read_csv("metinrag.csv")
+
+    if "checkpoint" in model_name_or_path:
+        try:
+            model.tokenizer.model_input_names.remove("token_type_ids")
+        except:
+            print(model_name_or_path)
+        df.question = df.question.apply(
+            lambda x: x.replace("İ", "i").replace("I", "ı").lower()
+        )
+        df.context = df.context.apply(
+            lambda x: x.replace("İ", "i").replace("I", "ı").lower()
+        )
+
+    # evaluate only first 1000 pairs
+    if not question2answer:
+        # filter long context rows
+        if shortContext:
+            df = df[
+                df.context.apply(
+                    lambda x: True if len(x.split()) < 400 else False
+                ).values
+                == True
+            ]
+        else:  # filter short context rows
+            df = df[
+                df.context.apply(
+                    lambda x: True if len(x.split()) > 400 else False
+                ).values
+                == True
+            ]
+    df = df.values[:1000]
+
+    # Read test queries
+    queries = {}
+    passage_cand = {}
+    relevant_qid = []
+    relevant_docs = defaultdict(lambda: defaultdict(int))
+
+    # df[i,2] for question2answer retrieval, df[i,3] for question2content retrieval
+    if question2answer:
+        candidatepassages = [[i, df[i, 2]] for i in range(len(df))]
+    else:
+        candidatepassages = [[i, df[i, 3]] for i in range(len(df))]
+
+    candidateIds = [[i for i in range(len(df))]]
+
+    for i, row in enumerate(df):
+        queries[str(i + 10000)] = row[1]
+        relevant_qid.append(str(i + 10000))
+        for z in range(len(df)):
+            relevant_docs[str(i + 10000)][str(z)] = 0
+        relevant_docs[str(i + 10000)][str(i)] = 1
+
+    queries_result_list = []
+    run = {}
+
+    documents_embeddings = model.encode(
+        [list(df[:, 2]) if question2answer else list(df[:, 3])],
+        is_query=False,
+    )
+
+    for qid in tqdm.tqdm(relevant_qid):
+        query = queries[qid]
+
+        queries_embeddings = model.encode(
+            [query],
+            is_query=True,
+        )
+
+        reranked_documents = rank.rerank(
+            documents_ids=candidateIds,
+            queries_embeddings=queries_embeddings,
+            documents_embeddings=documents_embeddings,
+        )
+
+        run[qid] = {}
+        for resDict in reranked_documents[0]:
+            run[qid][str(resDict["id"])] = float(resDict["score"])
+
+    evaluator = pytrec_eval.RelevanceEvaluator(
+        relevant_docs, pytrec_eval.supported_measures
+    )
+    scores = evaluator.evaluate(run)
+
+    def print_line(measure, scope, value):
+        print("{:25s}{:8s}{:.4f}".format(measure, scope, value))
+
+    for query_id, query_measures in sorted(scores.items()):
+        break
+        for measure, value in sorted(query_measures.items()):
+            print_line(measure, query_id, value)
+
+    # Scope hack: use query_measures of last item in previous loop to
+    # figure out all unique measure names.
+    resultsColumns = ["model name"]
+    resultsRow = [model_name_or_path]
+    for measure in sorted(query_measures.keys()):
+        resultsColumns.append(measure)
+        resultsRow.append(
+            pytrec_eval.compute_aggregated_measure(
+                measure, [query_measures[measure] for query_measures in scores.values()]
+            )
+        )
+
+    if evalResultsDf is None:
+        evalResultsDf = pd.DataFrame(columns=resultsColumns)
+    evalResultsDf.loc[-1] = resultsRow
+    evalResultsDf.index = evalResultsDf.index + 1
+
+evalResultsDf.to_csv("testResults.csv", encoding="utf-8")

assets/testResultsQ2A.csv

@@ -0,0 +1,4 @@
+,model name,11pt_avg,G,P_10,P_100,P_1000,P_15,P_20,P_200,P_30,P_5,P_500,Rndcg,Rprec,Rprec_mult_0.20,Rprec_mult_0.40,Rprec_mult_0.60,Rprec_mult_0.80,Rprec_mult_1.00,Rprec_mult_1.20,Rprec_mult_1.40,Rprec_mult_1.60,Rprec_mult_1.80,Rprec_mult_2.00,binG,bpref,gm_bpref,gm_map,infAP,iprec_at_recall_0.00,iprec_at_recall_0.10,iprec_at_recall_0.20,iprec_at_recall_0.30,iprec_at_recall_0.40,iprec_at_recall_0.50,iprec_at_recall_0.60,iprec_at_recall_0.70,iprec_at_recall_0.80,iprec_at_recall_0.90,iprec_at_recall_1.00,map,map_cut_10,map_cut_100,map_cut_1000,map_cut_15,map_cut_20,map_cut_200,map_cut_30,map_cut_5,map_cut_500,ndcg,ndcg_cut_10,ndcg_cut_100,ndcg_cut_1000,ndcg_cut_15,ndcg_cut_20,ndcg_cut_200,ndcg_cut_30,ndcg_cut_5,ndcg_cut_500,ndcg_rel,num_nonrel_judged_ret,num_q,num_rel,num_rel_ret,num_ret,recall_10,recall_100,recall_1000,recall_15,recall_20,recall_200,recall_30,recall_5,recall_500,recip_rank,relative_P_10,relative_P_100,relative_P_1000,relative_P_15,relative_P_20,relative_P_200,relative_P_30,relative_P_5,relative_P_500,relstring,runid,set_F,set_P,set_map,set_recall,set_relative_P,success_1,success_10,success_5,utility
+2,Y-J-Ju/ModernBERT-base-ColBERT,0.9246124301124922,0.9411712364322513,0.09750000000000002,0.00993,0.0010000000000000005,0.06533333333333333,0.04905000000000001,0.004979999999999999,0.03286666666666666,0.19300000000000003,0.001994000000000001,0.9165856182161257,0.892,0.892,0.892,0.892,0.892,0.892,0.468,0.468,0.468,0.468,0.468,0.9411712364322513,0.892,0.28840315031266056,0.8289736719061171,0.9246127562315869,0.9246124301124922,0.9246124301124922,0.9246124301124922,0.9246124301124922,0.9246124301124922,0.9246124301124922,0.9246124301124922,0.9246124301124922,0.9246124301124922,0.9246124301124922,0.9246124301124922,0.9246124301124922,0.9237769841269842,0.9245850056253159,0.9246124301124922,0.9241690587190587,0.9242315587190587,0.9246065845726843,0.9244316215447888,0.9225333333333333,0.9246086170930095,0.9411712364322513,0.9364491849637135,0.9403282992977279,0.9411712364322513,0.9377697203900529,0.9380143709321711,0.9407487743223146,0.9390770551961086,0.933304732232845,0.940860563074029,0.9411712364322513,999000.0,1000.0,1000.0,1000.0,1000000.0,0.975,0.993,1.0,0.98,0.981,0.996,0.986,0.965,0.997,0.9246124301124922,0.975,0.993,1.0,0.98,0.981,0.996,0.986,0.965,0.997,0.0,0.0,0.001998001998001999,0.0010000000000000005,0.0010000000000000005,1.0,1.0,0.892,0.975,0.965,-998.0
+1,99eren99/ColBERT-ModernBERT-base-Turkish-uncased,0.6979618587273602,0.752402836533961,0.08059999999999999,0.00894,0.0010000000000000005,0.05493333333333332,0.0417,0.004584999999999999,0.028399999999999998,0.1536,0.0019240000000000008,0.6927014182669805,0.633,0.633,0.633,0.633,0.633,0.633,0.356,0.356,0.356,0.356,0.356,0.752402836533961,0.633,0.01462177174456718,0.3265891980532749,0.6979625083364401,0.6979618587273602,0.6979618587273602,0.6979618587273602,0.6979618587273602,0.6979618587273602,0.6979618587273602,0.6979618587273602,0.6979618587273602,0.6979618587273602,0.6979618587273602,0.6979618587273602,0.6979618587273602,0.6939853174603174,0.6975972292535491,0.6979618587273602,0.6953998196248196,0.6959470934466291,0.6977587085420798,0.696698782271159,0.6885333333333333,0.6979078837145676,0.752402836533961,0.721298681712924,0.7397522621978438,0.752402836533961,0.7260568859493519,0.7283985537990932,0.7429533727644111,0.7322743488752352,0.7086283032247526,0.7483948932452603,0.752402836533961,999000.0,1000.0,1000.0,1000.0,1000000.0,0.806,0.894,1.0,0.824,0.834,0.917,0.852,0.768,0.962,0.6979618587273602,0.806,0.894,1.0,0.824,0.834,0.917,0.852,0.768,0.962,0.0,0.0,0.001998001998001999,0.0010000000000000005,0.0010000000000000005,1.0,1.0,0.633,0.806,0.768,-998.0
+0,jinaai/jina-colbert-v2,0.9869694444444445,0.9901166919324578,0.09980000000000001,0.01,0.0010000000000000005,0.06659999999999999,0.04995,0.005,0.033333333333333326,0.19920000000000004,0.002000000000000001,0.9845583459662289,0.979,0.979,0.979,0.979,0.979,0.979,0.496,0.496,0.496,0.496,0.496,0.9901166919324578,0.979,0.7852356346100717,0.9761456259363186,0.9869695241374947,0.9869694444444445,0.9869694444444445,0.9869694444444445,0.9869694444444445,0.9869694444444445,0.9869694444444445,0.9869694444444445,0.9869694444444445,0.9869694444444445,0.9869694444444445,0.9869694444444445,0.9869694444444445,0.9868611111111111,0.9869694444444445,0.9869694444444445,0.9869277777777778,0.9869277777777778,0.9869694444444445,0.9869694444444445,0.9865833333333334,0.9869694444444445,0.9901166919324578,0.989651353653421,0.9901166919324578,0.9901166919324578,0.9899013536534211,0.9899013536534211,0.9901166919324578,0.9901166919324578,0.988994116470649,0.9901166919324578,0.9901166919324578,999000.0,1000.0,1000.0,1000.0,1000000.0,0.998,1.0,1.0,0.999,0.999,1.0,1.0,0.996,1.0,0.9869694444444445,0.998,1.0,1.0,0.999,0.999,1.0,1.0,0.996,1.0,0.0,0.0,0.001998001998001999,0.0010000000000000005,0.0010000000000000005,1.0,1.0,0.979,0.998,0.996,-998.0

assets/testResultsQ2C_split_ge_400.csv

@@ -0,0 +1,4 @@
+,model name,11pt_avg,G,P_10,P_100,P_1000,P_15,P_20,P_200,P_30,P_5,P_500,Rndcg,Rprec,Rprec_mult_0.20,Rprec_mult_0.40,Rprec_mult_0.60,Rprec_mult_0.80,Rprec_mult_1.00,Rprec_mult_1.20,Rprec_mult_1.40,Rprec_mult_1.60,Rprec_mult_1.80,Rprec_mult_2.00,binG,bpref,gm_bpref,gm_map,infAP,iprec_at_recall_0.00,iprec_at_recall_0.10,iprec_at_recall_0.20,iprec_at_recall_0.30,iprec_at_recall_0.40,iprec_at_recall_0.50,iprec_at_recall_0.60,iprec_at_recall_0.70,iprec_at_recall_0.80,iprec_at_recall_0.90,iprec_at_recall_1.00,map,map_cut_10,map_cut_100,map_cut_1000,map_cut_15,map_cut_20,map_cut_200,map_cut_30,map_cut_5,map_cut_500,ndcg,ndcg_cut_10,ndcg_cut_100,ndcg_cut_1000,ndcg_cut_15,ndcg_cut_20,ndcg_cut_200,ndcg_cut_30,ndcg_cut_5,ndcg_cut_500,ndcg_rel,num_nonrel_judged_ret,num_q,num_rel,num_rel_ret,num_ret,recall_10,recall_100,recall_1000,recall_15,recall_20,recall_200,recall_30,recall_5,recall_500,recip_rank,relative_P_10,relative_P_100,relative_P_1000,relative_P_15,relative_P_20,relative_P_200,relative_P_30,relative_P_5,relative_P_500,relstring,runid,set_F,set_P,set_map,set_recall,set_relative_P,success_1,success_10,success_5,utility
+2,Y-J-Ju/ModernBERT-base-ColBERT,0.685448123051328,0.7565855383337551,0.09150000000000001,0.00983,0.0010000000000000005,0.06233333333333333,0.046950000000000006,0.00496,0.031799999999999995,0.17320000000000002,0.0019920000000000007,0.6472927691668776,0.538,0.538,0.538,0.538,0.538,0.538,0.37,0.37,0.37,0.37,0.37,0.7565855383337551,0.538,0.004897788193684461,0.4877016101617956,0.6854495975088933,0.685448123051328,0.685448123051328,0.685448123051328,0.685448123051328,0.685448123051328,0.685448123051328,0.685448123051328,0.685448123051328,0.685448123051328,0.685448123051328,0.685448123051328,0.685448123051328,0.6824369047619048,0.6853640499137148,0.685448123051328,0.6840181901431902,0.6842325692281575,0.6854286076928449,0.6848241651121445,0.6757833333333333,0.6854410976609923,0.7565855383337551,0.7400317717358111,0.7544138211635303,0.7565855383337551,0.745331869321164,0.7462624311984928,0.7556709924822292,0.7494378102934572,0.7240724877859027,0.7561488019222415,0.7565855383337551,999000.0,1000.0,1000.0,1000.0,1000000.0,0.915,0.983,1.0,0.935,0.939,0.992,0.954,0.866,0.996,0.685448123051328,0.915,0.983,1.0,0.935,0.939,0.992,0.954,0.866,0.996,0.0,0.0,0.001998001998001999,0.0010000000000000005,0.0010000000000000005,1.0,1.0,0.538,0.915,0.866,-998.0
+1,99eren99/ColBERT-ModernBERT-base-Turkish-uncased,0.7828840840039591,0.8356288676992767,0.09760000000000002,0.009959999999999998,0.0010000000000000005,0.06533333333333331,0.049150000000000006,0.004985,0.0329,0.19180000000000003,0.002000000000000001,0.7403144338496384,0.645,0.645,0.645,0.645,0.645,0.645,0.423,0.423,0.423,0.423,0.423,0.8356288676992767,0.645,0.016788040181225605,0.6692463684940848,0.7828854628215163,0.782884084003959,0.782884084003959,0.782884084003959,0.782884084003959,0.782884084003959,0.782884084003959,0.782884084003959,0.782884084003959,0.782884084003959,0.782884084003959,0.782884084003959,0.782884084003959,0.7820936507936508,0.7828691181501903,0.782884084003959,0.7823845598845598,0.7825610304727952,0.7828753293303147,0.7827268553886201,0.7797000000000001,0.782884084003959,0.8356288676992767,0.831043207950389,0.8351370210492257,0.8356288676992767,0.8320721508960401,0.8327915882957445,0.8352732636154372,0.833651344180952,0.8254196471696189,0.8356288676992767,0.8356288676992767,999000.0,1000.0,1000.0,1000.0,1000000.0,0.976,0.996,1.0,0.98,0.983,0.997,0.987,0.959,1.0,0.782884084003959,0.976,0.996,1.0,0.98,0.983,0.997,0.987,0.959,1.0,0.0,0.0,0.001998001998001999,0.0010000000000000005,0.0010000000000000005,1.0,1.0,0.645,0.976,0.959,-998.0
+0,jinaai/jina-colbert-v2,0.530716636371578,0.6206685439678394,0.07379999999999999,0.008649999999999998,0.0010000000000000005,0.051,0.039299999999999995,0.004524999999999999,0.027166666666666665,0.13839999999999997,0.0019160000000000008,0.5113342719839198,0.402,0.402,0.402,0.402,0.402,0.402,0.2795,0.2795,0.2795,0.2795,0.2795,0.6206685439678394,0.402,0.001023292992280756,0.20775454568036272,0.5307179235187451,0.530716636371578,0.530716636371578,0.530716636371578,0.530716636371578,0.530716636371578,0.530716636371578,0.530716636371578,0.530716636371578,0.530716636371578,0.530716636371578,0.530716636371578,0.530716636371578,0.5246051587301588,0.5302003273074456,0.530716636371578,0.5267890914640915,0.5279813816257701,0.5304827767986128,0.5292110555811114,0.5183833333333333,0.530658338651244,0.6206685439678394,0.577091010380624,0.6042743008500667,0.6206685439678394,0.584304211191409,0.589280716187244,0.609851163211391,0.5955524507019124,0.5621368244492102,0.6162547049512855,0.6206685439678394,999000.0,1000.0,1000.0,1000.0,1000000.0,0.738,0.865,1.0,0.765,0.786,0.905,0.815,0.692,0.958,0.530716636371578,0.738,0.865,1.0,0.765,0.786,0.905,0.815,0.692,0.958,0.0,0.0,0.001998001998001999,0.0010000000000000005,0.0010000000000000005,1.0,1.0,0.402,0.738,0.692,-998.0

assets/testResultsQ2C_split_le_400.csv

@@ -0,0 +1,4 @@
+,model name,11pt_avg,G,P_10,P_100,P_1000,P_15,P_20,P_200,P_30,P_5,P_500,Rndcg,Rprec,Rprec_mult_0.20,Rprec_mult_0.40,Rprec_mult_0.60,Rprec_mult_0.80,Rprec_mult_1.00,Rprec_mult_1.20,Rprec_mult_1.40,Rprec_mult_1.60,Rprec_mult_1.80,Rprec_mult_2.00,binG,bpref,gm_bpref,gm_map,infAP,iprec_at_recall_0.00,iprec_at_recall_0.10,iprec_at_recall_0.20,iprec_at_recall_0.30,iprec_at_recall_0.40,iprec_at_recall_0.50,iprec_at_recall_0.60,iprec_at_recall_0.70,iprec_at_recall_0.80,iprec_at_recall_0.90,iprec_at_recall_1.00,map,map_cut_10,map_cut_100,map_cut_1000,map_cut_15,map_cut_20,map_cut_200,map_cut_30,map_cut_5,map_cut_500,ndcg,ndcg_cut_10,ndcg_cut_100,ndcg_cut_1000,ndcg_cut_15,ndcg_cut_20,ndcg_cut_200,ndcg_cut_30,ndcg_cut_5,ndcg_cut_500,ndcg_rel,num_nonrel_judged_ret,num_q,num_rel,num_rel_ret,num_ret,recall_10,recall_100,recall_1000,recall_15,recall_20,recall_200,recall_30,recall_5,recall_500,recip_rank,relative_P_10,relative_P_100,relative_P_1000,relative_P_15,relative_P_20,relative_P_200,relative_P_30,relative_P_5,relative_P_500,relstring,runid,set_F,set_P,set_map,set_recall,set_relative_P,success_1,success_10,success_5,utility
+2,Y-J-Ju/ModernBERT-base-ColBERT,0.7603169824322866,0.8145966514811457,0.09300000000000001,0.009859999999999999,0.0010000000000000005,0.06313333333333332,0.04755000000000001,0.00496,0.0321,0.1806,0.001998000000000001,0.7297983257405729,0.645,0.645,0.645,0.645,0.645,0.645,0.403,0.403,0.403,0.403,0.403,0.8145966514811457,0.645,0.016788040181225605,0.580052989174925,0.760318135583314,0.7603169824322866,0.7603169824322866,0.7603169824322866,0.7603169824322866,0.7603169824322866,0.7603169824322866,0.7603169824322866,0.7603169824322866,0.7603169824322866,0.7603169824322866,0.7603169824322866,0.7603169824322866,0.7576904761904761,0.7602493073813712,0.7603169824322866,0.7590289543789542,0.7592426970690128,0.7602914597422048,0.7597206547584998,0.7541500000000001,0.7603150740353402,0.8145966514811457,0.8006452008635012,0.8128002601977398,0.8145966514811457,0.8051442676915247,0.8060741339801146,0.8136370411151292,0.808621242645529,0.7919776895352395,0.8144859851700634,0.8145966514811457,999000.0,1000.0,1000.0,1000.0,1000000.0,0.93,0.986,1.0,0.947,0.951,0.992,0.963,0.903,0.999,0.7603169824322866,0.93,0.986,1.0,0.947,0.951,0.992,0.963,0.903,0.999,0.0,0.0,0.001998001998001999,0.0010000000000000005,0.0010000000000000005,1.0,1.0,0.645,0.93,0.903,-998.0
+1,99eren99/ColBERT-ModernBERT-base-Turkish-uncased,0.8580457187638504,0.8927242042179343,0.09800000000000002,0.009959999999999998,0.0010000000000000005,0.06573333333333332,0.049400000000000006,0.00499,0.033,0.19500000000000003,0.001998000000000001,0.8258621021089672,0.759,0.759,0.759,0.759,0.759,0.759,0.4605,0.4605,0.4605,0.4605,0.4605,0.8927242042179343,0.759,0.06237348354824193,0.7720175058079288,0.858046709203294,0.8580457187638504,0.8580457187638504,0.8580457187638504,0.8580457187638504,0.8580457187638504,0.8580457187638504,0.8580457187638504,0.8580457187638504,0.8580457187638504,0.8580457187638504,0.8580457187638504,0.8580457187638504,0.8572107142857142,0.8580255605123029,0.8580457187638504,0.8576778138528138,0.8577991373822256,0.8580390746427873,0.8578790746863008,0.8564666666666667,0.8580439766035716,0.8927242042179343,0.8886206959220635,0.8922078631424599,0.8927242042179343,0.8902020312261889,0.8906864942348752,0.8924849051029011,0.8911113540114233,0.8869261500043951,0.8926151222513335,0.8927242042179343,999000.0,1000.0,1000.0,1000.0,1000000.0,0.98,0.996,1.0,0.986,0.988,0.998,0.99,0.975,0.999,0.8580457187638504,0.98,0.996,1.0,0.986,0.988,0.998,0.99,0.975,0.999,0.0,0.0,0.001998001998001999,0.0010000000000000005,0.0010000000000000005,1.0,1.0,0.759,0.98,0.975,-998.0
+0,jinaai/jina-colbert-v2,0.6347043671768042,0.6946566429494866,0.0729,0.008239999999999999,0.0010000000000000005,0.04953333333333333,0.0379,0.00431,0.02583333333333333,0.1402,0.0018420000000000008,0.6333283214747433,0.572,0.572,0.572,0.572,0.572,0.572,0.329,0.329,0.329,0.329,0.329,0.6946566429494866,0.572,0.007244359600749901,0.21082905547900538,0.6347049942107315,0.6347043671768041,0.6347043671768041,0.6347043671768041,0.6347043671768041,0.6347043671768041,0.6347043671768041,0.6347043671768041,0.6347043671768041,0.6347043671768041,0.6347043671768041,0.6347043671768041,0.6347043671768041,0.6306555555555555,0.6341262502981129,0.6347043671768041,0.6317422521922522,0.632606012082173,0.6344051604309523,0.6333007902458717,0.6269666666666666,0.6345933746318079,0.6946566429494866,0.6547815386257165,0.6739081228131658,0.6946566429494866,0.6584682285084629,0.6620392113134246,0.6792504270945724,0.6656749644703361,0.6457711058852837,0.6863393542584116,0.6946566429494866,999000.0,1000.0,1000.0,1000.0,1000000.0,0.729,0.824,1.0,0.743,0.758,0.862,0.775,0.701,0.921,0.6347043671768041,0.729,0.824,1.0,0.743,0.758,0.862,0.775,0.701,0.921,0.0,0.0,0.001998001998001999,0.0010000000000000005,0.0010000000000000005,1.0,1.0,0.572,0.729,0.701,-998.0