|
|
--- |
|
|
model-index: |
|
|
- name: XYZ-embedding-zh-v2 |
|
|
results: |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB CMedQAv1 |
|
|
revision: None |
|
|
split: test |
|
|
type: C-MTEB/CMedQAv1 |
|
|
metrics: |
|
|
- type: map |
|
|
value: 89.9766367822762 |
|
|
- type: mrr |
|
|
value: 91.88896825396824 |
|
|
- type: main_score |
|
|
value: 89.9766367822762 |
|
|
task: |
|
|
type: Reranking |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB CMedQAv2 |
|
|
revision: None |
|
|
split: test |
|
|
type: C-MTEB/CMedQAv2 |
|
|
metrics: |
|
|
- type: map |
|
|
value: 89.04628340075982 |
|
|
- type: mrr |
|
|
value: 91.21702380952381 |
|
|
- type: main_score |
|
|
value: 89.04628340075982 |
|
|
task: |
|
|
type: Reranking |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB CmedqaRetrieval |
|
|
revision: None |
|
|
split: dev |
|
|
type: C-MTEB/CmedqaRetrieval |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 27.796 |
|
|
- type: map_at_10 |
|
|
value: 41.498000000000005 |
|
|
- type: map_at_100 |
|
|
value: 43.332 |
|
|
- type: map_at_1000 |
|
|
value: 43.429 |
|
|
- type: map_at_3 |
|
|
value: 37.172 |
|
|
- type: map_at_5 |
|
|
value: 39.617000000000004 |
|
|
- type: mrr_at_1 |
|
|
value: 42.111 |
|
|
- type: mrr_at_10 |
|
|
value: 50.726000000000006 |
|
|
- type: mrr_at_100 |
|
|
value: 51.632 |
|
|
- type: mrr_at_1000 |
|
|
value: 51.67 |
|
|
- type: mrr_at_3 |
|
|
value: 48.429 |
|
|
- type: mrr_at_5 |
|
|
value: 49.662 |
|
|
- type: ndcg_at_1 |
|
|
value: 42.111 |
|
|
- type: ndcg_at_10 |
|
|
value: 48.294 |
|
|
- type: ndcg_at_100 |
|
|
value: 55.135999999999996 |
|
|
- type: ndcg_at_1000 |
|
|
value: 56.818000000000005 |
|
|
- type: ndcg_at_3 |
|
|
value: 43.185 |
|
|
- type: ndcg_at_5 |
|
|
value: 45.266 |
|
|
- type: precision_at_1 |
|
|
value: 42.111 |
|
|
- type: precision_at_10 |
|
|
value: 10.635 |
|
|
- type: precision_at_100 |
|
|
value: 1.619 |
|
|
- type: precision_at_1000 |
|
|
value: 0.183 |
|
|
- type: precision_at_3 |
|
|
value: 24.539 |
|
|
- type: precision_at_5 |
|
|
value: 17.644000000000002 |
|
|
- type: recall_at_1 |
|
|
value: 27.796 |
|
|
- type: recall_at_10 |
|
|
value: 59.034 |
|
|
- type: recall_at_100 |
|
|
value: 86.991 |
|
|
- type: recall_at_1000 |
|
|
value: 98.304 |
|
|
- type: recall_at_3 |
|
|
value: 43.356 |
|
|
- type: recall_at_5 |
|
|
value: 49.998 |
|
|
- type: main_score |
|
|
value: 48.294 |
|
|
task: |
|
|
type: Retrieval |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB CovidRetrieval |
|
|
revision: None |
|
|
split: dev |
|
|
type: C-MTEB/CovidRetrieval |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 80.479 |
|
|
- type: map_at_10 |
|
|
value: 87.984 |
|
|
- type: map_at_100 |
|
|
value: 88.036 |
|
|
- type: map_at_1000 |
|
|
value: 88.03699999999999 |
|
|
- type: map_at_3 |
|
|
value: 87.083 |
|
|
- type: map_at_5 |
|
|
value: 87.694 |
|
|
- type: mrr_at_1 |
|
|
value: 80.927 |
|
|
- type: mrr_at_10 |
|
|
value: 88.046 |
|
|
- type: mrr_at_100 |
|
|
value: 88.099 |
|
|
- type: mrr_at_1000 |
|
|
value: 88.1 |
|
|
- type: mrr_at_3 |
|
|
value: 87.215 |
|
|
- type: mrr_at_5 |
|
|
value: 87.768 |
|
|
- type: ndcg_at_1 |
|
|
value: 80.927 |
|
|
- type: ndcg_at_10 |
|
|
value: 90.756 |
|
|
- type: ndcg_at_100 |
|
|
value: 90.96 |
|
|
- type: ndcg_at_1000 |
|
|
value: 90.975 |
|
|
- type: ndcg_at_3 |
|
|
value: 89.032 |
|
|
- type: ndcg_at_5 |
|
|
value: 90.106 |
|
|
- type: precision_at_1 |
|
|
value: 80.927 |
|
|
- type: precision_at_10 |
|
|
value: 10.011000000000001 |
|
|
- type: precision_at_100 |
|
|
value: 1.009 |
|
|
- type: precision_at_1000 |
|
|
value: 0.101 |
|
|
- type: precision_at_3 |
|
|
value: 31.752999999999997 |
|
|
- type: precision_at_5 |
|
|
value: 19.6 |
|
|
- type: recall_at_1 |
|
|
value: 80.479 |
|
|
- type: recall_at_10 |
|
|
value: 99.05199999999999 |
|
|
- type: recall_at_100 |
|
|
value: 99.895 |
|
|
- type: recall_at_1000 |
|
|
value: 100.0 |
|
|
- type: recall_at_3 |
|
|
value: 94.494 |
|
|
- type: recall_at_5 |
|
|
value: 97.102 |
|
|
- type: main_score |
|
|
value: 90.756 |
|
|
task: |
|
|
type: Retrieval |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB DuRetrieval |
|
|
revision: None |
|
|
split: dev |
|
|
type: C-MTEB/DuRetrieval |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 27.853 |
|
|
- type: map_at_10 |
|
|
value: 85.13199999999999 |
|
|
- type: map_at_100 |
|
|
value: 87.688 |
|
|
- type: map_at_1000 |
|
|
value: 87.712 |
|
|
- type: map_at_3 |
|
|
value: 59.705 |
|
|
- type: map_at_5 |
|
|
value: 75.139 |
|
|
- type: mrr_at_1 |
|
|
value: 93.65 |
|
|
- type: mrr_at_10 |
|
|
value: 95.682 |
|
|
- type: mrr_at_100 |
|
|
value: 95.722 |
|
|
- type: mrr_at_1000 |
|
|
value: 95.724 |
|
|
- type: mrr_at_3 |
|
|
value: 95.467 |
|
|
- type: mrr_at_5 |
|
|
value: 95.612 |
|
|
- type: ndcg_at_1 |
|
|
value: 93.65 |
|
|
- type: ndcg_at_10 |
|
|
value: 91.155 |
|
|
- type: ndcg_at_100 |
|
|
value: 93.183 |
|
|
- type: ndcg_at_1000 |
|
|
value: 93.38499999999999 |
|
|
- type: ndcg_at_3 |
|
|
value: 90.648 |
|
|
- type: ndcg_at_5 |
|
|
value: 89.47699999999999 |
|
|
- type: precision_at_1 |
|
|
value: 93.65 |
|
|
- type: precision_at_10 |
|
|
value: 43.11 |
|
|
- type: precision_at_100 |
|
|
value: 4.854 |
|
|
- type: precision_at_1000 |
|
|
value: 0.49100000000000005 |
|
|
- type: precision_at_3 |
|
|
value: 81.11699999999999 |
|
|
- type: precision_at_5 |
|
|
value: 68.28999999999999 |
|
|
- type: recall_at_1 |
|
|
value: 27.853 |
|
|
- type: recall_at_10 |
|
|
value: 91.678 |
|
|
- type: recall_at_100 |
|
|
value: 98.553 |
|
|
- type: recall_at_1000 |
|
|
value: 99.553 |
|
|
- type: recall_at_3 |
|
|
value: 61.381 |
|
|
- type: recall_at_5 |
|
|
value: 78.605 |
|
|
- type: main_score |
|
|
value: 91.155 |
|
|
task: |
|
|
type: Retrieval |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB EcomRetrieval |
|
|
revision: None |
|
|
split: dev |
|
|
type: C-MTEB/EcomRetrieval |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 54.50000000000001 |
|
|
- type: map_at_10 |
|
|
value: 65.167 |
|
|
- type: map_at_100 |
|
|
value: 65.664 |
|
|
- type: map_at_1000 |
|
|
value: 65.67399999999999 |
|
|
- type: map_at_3 |
|
|
value: 62.633 |
|
|
- type: map_at_5 |
|
|
value: 64.208 |
|
|
- type: mrr_at_1 |
|
|
value: 54.50000000000001 |
|
|
- type: mrr_at_10 |
|
|
value: 65.167 |
|
|
- type: mrr_at_100 |
|
|
value: 65.664 |
|
|
- type: mrr_at_1000 |
|
|
value: 65.67399999999999 |
|
|
- type: mrr_at_3 |
|
|
value: 62.633 |
|
|
- type: mrr_at_5 |
|
|
value: 64.208 |
|
|
- type: ndcg_at_1 |
|
|
value: 54.50000000000001 |
|
|
- type: ndcg_at_10 |
|
|
value: 70.294 |
|
|
- type: ndcg_at_100 |
|
|
value: 72.564 |
|
|
- type: ndcg_at_1000 |
|
|
value: 72.841 |
|
|
- type: ndcg_at_3 |
|
|
value: 65.128 |
|
|
- type: ndcg_at_5 |
|
|
value: 67.96799999999999 |
|
|
- type: precision_at_1 |
|
|
value: 54.50000000000001 |
|
|
- type: precision_at_10 |
|
|
value: 8.64 |
|
|
- type: precision_at_100 |
|
|
value: 0.967 |
|
|
- type: precision_at_1000 |
|
|
value: 0.099 |
|
|
- type: precision_at_3 |
|
|
value: 24.099999999999998 |
|
|
- type: precision_at_5 |
|
|
value: 15.840000000000002 |
|
|
- type: recall_at_1 |
|
|
value: 54.50000000000001 |
|
|
- type: recall_at_10 |
|
|
value: 86.4 |
|
|
- type: recall_at_100 |
|
|
value: 96.7 |
|
|
- type: recall_at_1000 |
|
|
value: 98.9 |
|
|
- type: recall_at_3 |
|
|
value: 72.3 |
|
|
- type: recall_at_5 |
|
|
value: 79.2 |
|
|
- type: main_score |
|
|
value: 70.294 |
|
|
task: |
|
|
type: Retrieval |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB MMarcoReranking |
|
|
revision: None |
|
|
split: dev |
|
|
type: C-MTEB/Mmarco-reranking |
|
|
metrics: |
|
|
- type: map |
|
|
value: 37.68251937316638 |
|
|
- type: mrr |
|
|
value: 36.61746031746032 |
|
|
- type: main_score |
|
|
value: 37.68251937316638 |
|
|
task: |
|
|
type: Reranking |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB MMarcoRetrieval |
|
|
revision: None |
|
|
split: dev |
|
|
type: C-MTEB/MMarcoRetrieval |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 69.401 |
|
|
- type: map_at_10 |
|
|
value: 78.8 |
|
|
- type: map_at_100 |
|
|
value: 79.077 |
|
|
- type: map_at_1000 |
|
|
value: 79.081 |
|
|
- type: map_at_3 |
|
|
value: 76.97 |
|
|
- type: map_at_5 |
|
|
value: 78.185 |
|
|
- type: mrr_at_1 |
|
|
value: 71.719 |
|
|
- type: mrr_at_10 |
|
|
value: 79.327 |
|
|
- type: mrr_at_100 |
|
|
value: 79.56400000000001 |
|
|
- type: mrr_at_1000 |
|
|
value: 79.56800000000001 |
|
|
- type: mrr_at_3 |
|
|
value: 77.736 |
|
|
- type: mrr_at_5 |
|
|
value: 78.782 |
|
|
- type: ndcg_at_1 |
|
|
value: 71.719 |
|
|
- type: ndcg_at_10 |
|
|
value: 82.505 |
|
|
- type: ndcg_at_100 |
|
|
value: 83.673 |
|
|
- type: ndcg_at_1000 |
|
|
value: 83.786 |
|
|
- type: ndcg_at_3 |
|
|
value: 79.07600000000001 |
|
|
- type: ndcg_at_5 |
|
|
value: 81.122 |
|
|
- type: precision_at_1 |
|
|
value: 71.719 |
|
|
- type: precision_at_10 |
|
|
value: 9.924 |
|
|
- type: precision_at_100 |
|
|
value: 1.049 |
|
|
- type: precision_at_1000 |
|
|
value: 0.106 |
|
|
- type: precision_at_3 |
|
|
value: 29.742 |
|
|
- type: precision_at_5 |
|
|
value: 18.937 |
|
|
- type: recall_at_1 |
|
|
value: 69.401 |
|
|
- type: recall_at_10 |
|
|
value: 93.349 |
|
|
- type: recall_at_100 |
|
|
value: 98.492 |
|
|
- type: recall_at_1000 |
|
|
value: 99.384 |
|
|
- type: recall_at_3 |
|
|
value: 84.385 |
|
|
- type: recall_at_5 |
|
|
value: 89.237 |
|
|
- type: main_score |
|
|
value: 82.505 |
|
|
task: |
|
|
type: Retrieval |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB MedicalRetrieval |
|
|
revision: None |
|
|
split: dev |
|
|
type: C-MTEB/MedicalRetrieval |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 57.8 |
|
|
- type: map_at_10 |
|
|
value: 64.696 |
|
|
- type: map_at_100 |
|
|
value: 65.294 |
|
|
- type: map_at_1000 |
|
|
value: 65.328 |
|
|
- type: map_at_3 |
|
|
value: 62.949999999999996 |
|
|
- type: map_at_5 |
|
|
value: 64.095 |
|
|
- type: mrr_at_1 |
|
|
value: 58.099999999999994 |
|
|
- type: mrr_at_10 |
|
|
value: 64.85 |
|
|
- type: mrr_at_100 |
|
|
value: 65.448 |
|
|
- type: mrr_at_1000 |
|
|
value: 65.482 |
|
|
- type: mrr_at_3 |
|
|
value: 63.1 |
|
|
- type: mrr_at_5 |
|
|
value: 64.23 |
|
|
- type: ndcg_at_1 |
|
|
value: 57.8 |
|
|
- type: ndcg_at_10 |
|
|
value: 68.041 |
|
|
- type: ndcg_at_100 |
|
|
value: 71.074 |
|
|
- type: ndcg_at_1000 |
|
|
value: 71.919 |
|
|
- type: ndcg_at_3 |
|
|
value: 64.584 |
|
|
- type: ndcg_at_5 |
|
|
value: 66.625 |
|
|
- type: precision_at_1 |
|
|
value: 57.8 |
|
|
- type: precision_at_10 |
|
|
value: 7.85 |
|
|
- type: precision_at_100 |
|
|
value: 0.9289999999999999 |
|
|
- type: precision_at_1000 |
|
|
value: 0.099 |
|
|
- type: precision_at_3 |
|
|
value: 23.1 |
|
|
- type: precision_at_5 |
|
|
value: 14.84 |
|
|
- type: recall_at_1 |
|
|
value: 57.8 |
|
|
- type: recall_at_10 |
|
|
value: 78.5 |
|
|
- type: recall_at_100 |
|
|
value: 92.9 |
|
|
- type: recall_at_1000 |
|
|
value: 99.4 |
|
|
- type: recall_at_3 |
|
|
value: 69.3 |
|
|
- type: recall_at_5 |
|
|
value: 74.2 |
|
|
- type: main_score |
|
|
value: 68.041 |
|
|
task: |
|
|
type: Retrieval |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB T2Reranking |
|
|
revision: None |
|
|
split: dev |
|
|
type: C-MTEB/T2Reranking |
|
|
metrics: |
|
|
- type: map |
|
|
value: 69.13287570713865 |
|
|
- type: mrr |
|
|
value: 79.95326487625066 |
|
|
- type: main_score |
|
|
value: 69.13287570713865 |
|
|
task: |
|
|
type: Reranking |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB T2Retrieval |
|
|
revision: None |
|
|
split: dev |
|
|
type: C-MTEB/T2Retrieval |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 28.041 |
|
|
- type: map_at_10 |
|
|
value: 78.509 |
|
|
- type: map_at_100 |
|
|
value: 82.083 |
|
|
- type: map_at_1000 |
|
|
value: 82.143 |
|
|
- type: map_at_3 |
|
|
value: 55.345 |
|
|
- type: map_at_5 |
|
|
value: 67.899 |
|
|
- type: mrr_at_1 |
|
|
value: 90.86 |
|
|
- type: mrr_at_10 |
|
|
value: 93.31 |
|
|
- type: mrr_at_100 |
|
|
value: 93.388 |
|
|
- type: mrr_at_1000 |
|
|
value: 93.391 |
|
|
- type: mrr_at_3 |
|
|
value: 92.92200000000001 |
|
|
- type: mrr_at_5 |
|
|
value: 93.167 |
|
|
- type: ndcg_at_1 |
|
|
value: 90.86 |
|
|
- type: ndcg_at_10 |
|
|
value: 85.875 |
|
|
- type: ndcg_at_100 |
|
|
value: 89.269 |
|
|
- type: ndcg_at_1000 |
|
|
value: 89.827 |
|
|
- type: ndcg_at_3 |
|
|
value: 87.254 |
|
|
- type: ndcg_at_5 |
|
|
value: 85.855 |
|
|
- type: precision_at_1 |
|
|
value: 90.86 |
|
|
- type: precision_at_10 |
|
|
value: 42.488 |
|
|
- type: precision_at_100 |
|
|
value: 5.029 |
|
|
- type: precision_at_1000 |
|
|
value: 0.516 |
|
|
- type: precision_at_3 |
|
|
value: 76.172 |
|
|
- type: precision_at_5 |
|
|
value: 63.759 |
|
|
- type: recall_at_1 |
|
|
value: 28.041 |
|
|
- type: recall_at_10 |
|
|
value: 84.829 |
|
|
- type: recall_at_100 |
|
|
value: 95.89999999999999 |
|
|
- type: recall_at_1000 |
|
|
value: 98.665 |
|
|
- type: recall_at_3 |
|
|
value: 57.009 |
|
|
- type: recall_at_5 |
|
|
value: 71.188 |
|
|
- type: main_score |
|
|
value: 85.875 |
|
|
task: |
|
|
type: Retrieval |
|
|
- dataset: |
|
|
config: default |
|
|
name: MTEB VideoRetrieval |
|
|
revision: None |
|
|
split: dev |
|
|
type: C-MTEB/VideoRetrieval |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 67.30000000000001 |
|
|
- type: map_at_10 |
|
|
value: 76.819 |
|
|
- type: map_at_100 |
|
|
value: 77.141 |
|
|
- type: map_at_1000 |
|
|
value: 77.142 |
|
|
- type: map_at_3 |
|
|
value: 75.233 |
|
|
- type: map_at_5 |
|
|
value: 76.163 |
|
|
- type: mrr_at_1 |
|
|
value: 67.30000000000001 |
|
|
- type: mrr_at_10 |
|
|
value: 76.819 |
|
|
- type: mrr_at_100 |
|
|
value: 77.141 |
|
|
- type: mrr_at_1000 |
|
|
value: 77.142 |
|
|
- type: mrr_at_3 |
|
|
value: 75.233 |
|
|
- type: mrr_at_5 |
|
|
value: 76.163 |
|
|
- type: ndcg_at_1 |
|
|
value: 67.30000000000001 |
|
|
- type: ndcg_at_10 |
|
|
value: 80.93599999999999 |
|
|
- type: ndcg_at_100 |
|
|
value: 82.311 |
|
|
- type: ndcg_at_1000 |
|
|
value: 82.349 |
|
|
- type: ndcg_at_3 |
|
|
value: 77.724 |
|
|
- type: ndcg_at_5 |
|
|
value: 79.406 |
|
|
- type: precision_at_1 |
|
|
value: 67.30000000000001 |
|
|
- type: precision_at_10 |
|
|
value: 9.36 |
|
|
- type: precision_at_100 |
|
|
value: 0.996 |
|
|
- type: precision_at_1000 |
|
|
value: 0.1 |
|
|
- type: precision_at_3 |
|
|
value: 28.299999999999997 |
|
|
- type: precision_at_5 |
|
|
value: 17.8 |
|
|
- type: recall_at_1 |
|
|
value: 67.30000000000001 |
|
|
- type: recall_at_10 |
|
|
value: 93.60000000000001 |
|
|
- type: recall_at_100 |
|
|
value: 99.6 |
|
|
- type: recall_at_1000 |
|
|
value: 99.9 |
|
|
- type: recall_at_3 |
|
|
value: 84.89999999999999 |
|
|
- type: recall_at_5 |
|
|
value: 89.0 |
|
|
- type: main_score |
|
|
value: 80.93599999999999 |
|
|
task: |
|
|
type: Retrieval |
|
|
tags: |
|
|
- mteb |
|
|
language: |
|
|
- zh |
|
|
|
|
|
--- |
|
|
|
|
|
<h2 align="left">XYZ-embedding-zh-v2</h2> |
|
|
|
|
|
## Usage (Sentence Transformers) |
|
|
|
|
|
First install the Sentence Transformers library: |
|
|
|
|
|
```bash |
|
|
pip install -U sentence-transformers |
|
|
``` |
|
|
Then you can load this model and run inference. |
|
|
```python |
|
|
from sentence_transformers import SentenceTransformer |
|
|
|
|
|
# Download from the 🤗 Hub |
|
|
model = SentenceTransformer("fangxq/XYZ-embedding-zh-v2") |
|
|
# Run inference |
|
|
sentences = [ |
|
|
'The weather is lovely today.', |
|
|
"It's so sunny outside!", |
|
|
'He drove to the stadium.', |
|
|
] |
|
|
embeddings = model.encode(sentences) |
|
|
print(embeddings.shape) |
|
|
# [3, 1792] |
|
|
|
|
|
# Get the similarity scores for the embeddings |
|
|
similarities = model.similarity(embeddings, embeddings) |
|
|
print(similarities.shape) |
|
|
# [3, 3] |
|
|
``` |
