| --- |
| model-index: |
| - name: Quark-Emb-1.5b |
| results: |
| - dataset: |
| config: default |
| name: MTEB AFQMC |
| revision: None |
| split: validation |
| type: C-MTEB/AFQMC |
| metrics: |
| - type: cosine_pearson |
| value: 47.14285927987258 |
| - type: cosine_spearman |
| value: 48.161200368263025 |
| - type: manhattan_pearson |
| value: 46.852921578928694 |
| - type: manhattan_spearman |
| value: 48.0768829644805 |
| - type: euclidean_pearson |
| value: 46.934710408297846 |
| - type: euclidean_spearman |
| value: 48.161200368263025 |
| - type: main_score |
| value: 48.161200368263025 |
| task: |
| type: STS |
| - dataset: |
| config: default |
| name: MTEB ATEC |
| revision: None |
| split: test |
| type: C-MTEB/ATEC |
| metrics: |
| - type: cosine_pearson |
| value: 53.31694395347832 |
| - type: cosine_spearman |
| value: 50.82142054857025 |
| - type: manhattan_pearson |
| value: 55.63018022546727 |
| - type: manhattan_spearman |
| value: 50.808925663235286 |
| - type: euclidean_pearson |
| value: 55.630897902214585 |
| - type: euclidean_spearman |
| value: 50.82142054857025 |
| - type: main_score |
| value: 50.82142054857025 |
| task: |
| type: STS |
| - dataset: |
| config: zh |
| name: MTEB AmazonReviewsClassification (zh) |
| revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
| split: test |
| type: mteb/amazon_reviews_multi |
| metrics: |
| - type: accuracy |
| value: 51.93800000000001 |
| - type: accuracy_stderr |
| value: 1.6225030046197138 |
| - type: f1 |
| value: 49.36480272612989 |
| - type: f1_stderr |
| value: 2.402473535325102 |
| - type: main_score |
| value: 51.93800000000001 |
| task: |
| type: Classification |
| - dataset: |
| config: zh |
| name: MTEB AmazonReviewsClassification (zh) |
| revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
| split: validation |
| type: mteb/amazon_reviews_multi |
| metrics: |
| - type: accuracy |
| value: 50.757999999999996 |
| - type: accuracy_stderr |
| value: 1.1949041802588176 |
| - type: f1 |
| value: 48.18542841607346 |
| - type: f1_stderr |
| value: 2.025507464835368 |
| - type: main_score |
| value: 50.757999999999996 |
| task: |
| type: Classification |
| - dataset: |
| config: default |
| name: MTEB BQ |
| revision: None |
| split: test |
| type: C-MTEB/BQ |
| metrics: |
| - type: cosine_pearson |
| value: 66.94471481392273 |
| - type: cosine_spearman |
| value: 67.86811107045457 |
| - type: manhattan_pearson |
| value: 65.56778188873142 |
| - type: manhattan_spearman |
| value: 67.83060870618156 |
| - type: euclidean_pearson |
| value: 65.63668085779311 |
| - type: euclidean_spearman |
| value: 67.86811107045457 |
| - type: main_score |
| value: 67.86811107045457 |
| task: |
| type: STS |
| - dataset: |
| config: default |
| name: MTEB CLSClusteringP2P |
| revision: None |
| split: test |
| type: C-MTEB/CLSClusteringP2P |
| metrics: |
| - type: v_measure |
| value: 58.53706905558472 |
| - type: v_measure_std |
| value: 1.3628784531981595 |
| - type: main_score |
| value: 58.53706905558472 |
| task: |
| type: Clustering |
| - dataset: |
| config: default |
| name: MTEB CLSClusteringS2S |
| revision: None |
| split: test |
| type: C-MTEB/CLSClusteringS2S |
| metrics: |
| - type: v_measure |
| value: 54.70969139354621 |
| - type: v_measure_std |
| value: 1.938384688132648 |
| - type: main_score |
| value: 54.70969139354621 |
| task: |
| type: Clustering |
| - dataset: |
| config: default |
| name: MTEB CMedQAv1 |
| revision: None |
| split: test |
| type: C-MTEB/CMedQAv1-reranking |
| metrics: |
| - type: map |
| value: 87.79521046311835 |
| - type: mrr |
| value: 90.01547619047618 |
| - type: main_score |
| value: 87.79521046311835 |
| task: |
| type: Reranking |
| - dataset: |
| config: default |
| name: MTEB CMedQAv2 |
| revision: None |
| split: test |
| type: C-MTEB/CMedQAv2-reranking |
| metrics: |
| - type: map |
| value: 87.89916670870878 |
| - type: mrr |
| value: 89.92595238095238 |
| - type: main_score |
| value: 87.89916670870878 |
| task: |
| type: Reranking |
| - dataset: |
| config: default |
| name: MTEB CmedqaRetrieval |
| revision: None |
| split: dev |
| type: C-MTEB/CmedqaRetrieval |
| metrics: |
| - type: map_at_1 |
| value: 25.444 |
| - type: map_at_10 |
| value: 37.763999999999996 |
| - type: map_at_100 |
| value: 39.641999999999996 |
| - type: map_at_1000 |
| value: 39.756 |
| - type: map_at_3 |
| value: 33.742 |
| - type: map_at_5 |
| value: 35.906 |
| - type: mrr_at_1 |
| value: 38.71 |
| - type: mrr_at_10 |
| value: 46.744 |
| - type: mrr_at_100 |
| value: 47.745 |
| - type: mrr_at_1000 |
| value: 47.791 |
| - type: mrr_at_3 |
| value: 44.324000000000005 |
| - type: mrr_at_5 |
| value: 45.696 |
| - type: ndcg_at_1 |
| value: 38.71 |
| - type: ndcg_at_10 |
| value: 44.285000000000004 |
| - type: ndcg_at_100 |
| value: 51.69200000000001 |
| - type: ndcg_at_1000 |
| value: 53.669999999999995 |
| - type: ndcg_at_3 |
| value: 39.273 |
| - type: ndcg_at_5 |
| value: 41.254000000000005 |
| - type: precision_at_1 |
| value: 38.71 |
| - type: precision_at_10 |
| value: 9.825000000000001 |
| - type: precision_at_100 |
| value: 1.583 |
| - type: precision_at_1000 |
| value: 0.183 |
| - type: precision_at_3 |
| value: 22.197 |
| - type: precision_at_5 |
| value: 16.019 |
| - type: recall_at_1 |
| value: 25.444 |
| - type: recall_at_10 |
| value: 54.535999999999994 |
| - type: recall_at_100 |
| value: 85.307 |
| - type: recall_at_1000 |
| value: 98.473 |
| - type: recall_at_3 |
| value: 39.274 |
| - type: recall_at_5 |
| value: 45.580999999999996 |
| - type: main_score |
| value: 44.285000000000004 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB Cmnli |
| revision: None |
| split: validation |
| type: C-MTEB/CMNLI |
| metrics: |
| - type: cos_sim_accuracy |
| value: 89.58508719182201 |
| - type: cos_sim_accuracy_threshold |
| value: 97.09511288861569 |
| - type: cos_sim_ap |
| value: 95.12338246323735 |
| - type: cos_sim_f1 |
| value: 90.19211324570271 |
| - type: cos_sim_f1_threshold |
| value: 97.02014138938755 |
| - type: cos_sim_precision |
| value: 86.80795847750865 |
| - type: cos_sim_recall |
| value: 93.85083002104278 |
| - type: dot_accuracy |
| value: 89.58508719182201 |
| - type: dot_accuracy_threshold |
| value: 97.0951128886157 |
| - type: dot_ap |
| value: 95.13959275940286 |
| - type: dot_f1 |
| value: 90.19211324570271 |
| - type: dot_f1_threshold |
| value: 97.02014138938755 |
| - type: dot_precision |
| value: 86.80795847750865 |
| - type: dot_recall |
| value: 93.85083002104278 |
| - type: euclidean_accuracy |
| value: 89.58508719182201 |
| - type: euclidean_accuracy_threshold |
| value: 24.103473235790947 |
| - type: euclidean_ap |
| value: 95.12338246323735 |
| - type: euclidean_f1 |
| value: 90.19211324570271 |
| - type: euclidean_f1_threshold |
| value: 24.412531977088996 |
| - type: euclidean_precision |
| value: 86.80795847750865 |
| - type: euclidean_recall |
| value: 93.85083002104278 |
| - type: manhattan_accuracy |
| value: 89.57306073361396 |
| - type: manhattan_accuracy_threshold |
| value: 729.1211254739587 |
| - type: manhattan_ap |
| value: 95.12388319543341 |
| - type: manhattan_f1 |
| value: 90.13956654941563 |
| - type: manhattan_f1_threshold |
| value: 733.155723492131 |
| - type: manhattan_precision |
| value: 87.56613756613757 |
| - type: manhattan_recall |
| value: 92.8688332943652 |
| - type: max_accuracy |
| value: 89.58508719182201 |
| - type: max_ap |
| value: 95.13959275940286 |
| - type: max_f1 |
| value: 90.19211324570271 |
| task: |
| type: PairClassification |
| - dataset: |
| config: default |
| name: MTEB CovidRetrieval |
| revision: None |
| split: dev |
| type: C-MTEB/CovidRetrieval |
| metrics: |
| - type: map_at_1 |
| value: 75.29 |
| - type: map_at_10 |
| value: 82.392 |
| - type: map_at_100 |
| value: 82.581 |
| - type: map_at_1000 |
| value: 82.585 |
| - type: map_at_3 |
| value: 80.88300000000001 |
| - type: map_at_5 |
| value: 81.71199999999999 |
| - type: mrr_at_1 |
| value: 75.553 |
| - type: mrr_at_10 |
| value: 82.422 |
| - type: mrr_at_100 |
| value: 82.6 |
| - type: mrr_at_1000 |
| value: 82.604 |
| - type: mrr_at_3 |
| value: 80.927 |
| - type: mrr_at_5 |
| value: 81.765 |
| - type: ndcg_at_1 |
| value: 75.44800000000001 |
| - type: ndcg_at_10 |
| value: 85.655 |
| - type: ndcg_at_100 |
| value: 86.435 |
| - type: ndcg_at_1000 |
| value: 86.541 |
| - type: ndcg_at_3 |
| value: 82.60300000000001 |
| - type: ndcg_at_5 |
| value: 84.062 |
| - type: precision_at_1 |
| value: 75.44800000000001 |
| - type: precision_at_10 |
| value: 9.663 |
| - type: precision_at_100 |
| value: 1.002 |
| - type: precision_at_1000 |
| value: 0.101 |
| - type: precision_at_3 |
| value: 29.329 |
| - type: precision_at_5 |
| value: 18.314 |
| - type: recall_at_1 |
| value: 75.29 |
| - type: recall_at_10 |
| value: 95.838 |
| - type: recall_at_100 |
| value: 99.157 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: recall_at_3 |
| value: 87.566 |
| - type: recall_at_5 |
| value: 90.991 |
| - type: main_score |
| value: 85.655 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB DuRetrieval |
| revision: None |
| split: dev |
| type: C-MTEB/DuRetrieval |
| metrics: |
| - type: map_at_1 |
| value: 27.584999999999997 |
| - type: map_at_10 |
| value: 85.112 |
| - type: map_at_100 |
| value: 87.632 |
| - type: map_at_1000 |
| value: 87.654 |
| - type: map_at_3 |
| value: 59.504999999999995 |
| - type: map_at_5 |
| value: 75.029 |
| - type: mrr_at_1 |
| value: 93.30000000000001 |
| - type: mrr_at_10 |
| value: 95.44200000000001 |
| - type: mrr_at_100 |
| value: 95.498 |
| - type: mrr_at_1000 |
| value: 95.5 |
| - type: mrr_at_3 |
| value: 95.258 |
| - type: mrr_at_5 |
| value: 95.36099999999999 |
| - type: ndcg_at_1 |
| value: 93.30000000000001 |
| - type: ndcg_at_10 |
| value: 91.086 |
| - type: ndcg_at_100 |
| value: 93.089 |
| - type: ndcg_at_1000 |
| value: 93.297 |
| - type: ndcg_at_3 |
| value: 90.432 |
| - type: ndcg_at_5 |
| value: 89.361 |
| - type: precision_at_1 |
| value: 93.30000000000001 |
| - type: precision_at_10 |
| value: 43.21 |
| - type: precision_at_100 |
| value: 4.857 |
| - type: precision_at_1000 |
| value: 0.49 |
| - type: precision_at_3 |
| value: 81.0 |
| - type: precision_at_5 |
| value: 68.28999999999999 |
| - type: recall_at_1 |
| value: 27.584999999999997 |
| - type: recall_at_10 |
| value: 91.73599999999999 |
| - type: recall_at_100 |
| value: 98.648 |
| - type: recall_at_1000 |
| value: 99.751 |
| - type: recall_at_3 |
| value: 61.378 |
| - type: recall_at_5 |
| value: 78.672 |
| - type: main_score |
| value: 91.086 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB EcomRetrieval |
| revision: None |
| split: dev |
| type: C-MTEB/EcomRetrieval |
| metrics: |
| - type: map_at_1 |
| value: 55.1 |
| - type: map_at_10 |
| value: 65.268 |
| - type: map_at_100 |
| value: 65.756 |
| - type: map_at_1000 |
| value: 65.765 |
| - type: map_at_3 |
| value: 63.132999999999996 |
| - type: map_at_5 |
| value: 64.25800000000001 |
| - type: mrr_at_1 |
| value: 55.1 |
| - type: mrr_at_10 |
| value: 65.268 |
| - type: mrr_at_100 |
| value: 65.756 |
| - type: mrr_at_1000 |
| value: 65.765 |
| - type: mrr_at_3 |
| value: 63.132999999999996 |
| - type: mrr_at_5 |
| value: 64.25800000000001 |
| - type: ndcg_at_1 |
| value: 55.1 |
| - type: ndcg_at_10 |
| value: 70.15599999999999 |
| - type: ndcg_at_100 |
| value: 72.368 |
| - type: ndcg_at_1000 |
| value: 72.635 |
| - type: ndcg_at_3 |
| value: 65.697 |
| - type: ndcg_at_5 |
| value: 67.741 |
| - type: precision_at_1 |
| value: 55.1 |
| - type: precision_at_10 |
| value: 8.55 |
| - type: precision_at_100 |
| value: 0.955 |
| - type: precision_at_1000 |
| value: 0.098 |
| - type: precision_at_3 |
| value: 24.367 |
| - type: precision_at_5 |
| value: 15.620000000000001 |
| - type: recall_at_1 |
| value: 55.1 |
| - type: recall_at_10 |
| value: 85.5 |
| - type: recall_at_100 |
| value: 95.5 |
| - type: recall_at_1000 |
| value: 97.6 |
| - type: recall_at_3 |
| value: 73.1 |
| - type: recall_at_5 |
| value: 78.10000000000001 |
| - type: main_score |
| value: 70.15599999999999 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB IFlyTek |
| revision: None |
| split: validation |
| type: C-MTEB/IFlyTek-classification |
| metrics: |
| - type: accuracy |
| value: 52.743362831858406 |
| - type: accuracy_stderr |
| value: 0.4449967616714387 |
| - type: f1 |
| value: 40.13427504900375 |
| - type: f1_stderr |
| value: 0.17565290177989018 |
| - type: main_score |
| value: 52.743362831858406 |
| task: |
| type: Classification |
| - dataset: |
| config: default |
| name: MTEB JDReview |
| revision: None |
| split: test |
| type: C-MTEB/JDReview-classification |
| metrics: |
| - type: accuracy |
| value: 90.13133208255161 |
| - type: accuracy_stderr |
| value: 0.9647249630155678 |
| - type: ap |
| value: 62.848199712439765 |
| - type: ap_stderr |
| value: 1.986859492917626 |
| - type: f1 |
| value: 85.48543445690254 |
| - type: f1_stderr |
| value: 1.0490059319804828 |
| - type: main_score |
| value: 90.13133208255161 |
| task: |
| type: Classification |
| - dataset: |
| config: default |
| name: MTEB LCQMC |
| revision: None |
| split: test |
| type: C-MTEB/LCQMC |
| metrics: |
| - type: cosine_pearson |
| value: 77.75677384428634 |
| - type: cosine_spearman |
| value: 78.86284859566986 |
| - type: manhattan_pearson |
| value: 79.8032754323316 |
| - type: manhattan_spearman |
| value: 78.85558562163624 |
| - type: euclidean_pearson |
| value: 79.82552324704292 |
| - type: euclidean_spearman |
| value: 78.86284859566986 |
| - type: main_score |
| value: 78.86284859566986 |
| task: |
| type: STS |
| - dataset: |
| config: default |
| name: MTEB MMarcoReranking |
| revision: None |
| split: dev |
| type: C-MTEB/Mmarco-reranking |
| metrics: |
| - type: map |
| value: 30.737025407798523 |
| - type: mrr |
| value: 29.26111111111111 |
| - type: main_score |
| value: 30.737025407798523 |
| task: |
| type: Reranking |
| - dataset: |
| config: default |
| name: MTEB MMarcoRetrieval |
| revision: None |
| split: dev |
| type: C-MTEB/MMarcoRetrieval |
| metrics: |
| - type: map_at_1 |
| value: 70.244 |
| - type: map_at_10 |
| value: 78.975 |
| - type: map_at_100 |
| value: 79.253 |
| - type: map_at_1000 |
| value: 79.26100000000001 |
| - type: map_at_3 |
| value: 77.363 |
| - type: map_at_5 |
| value: 78.364 |
| - type: mrr_at_1 |
| value: 72.521 |
| - type: mrr_at_10 |
| value: 79.514 |
| - type: mrr_at_100 |
| value: 79.75 |
| - type: mrr_at_1000 |
| value: 79.757 |
| - type: mrr_at_3 |
| value: 78.095 |
| - type: mrr_at_5 |
| value: 78.987 |
| - type: ndcg_at_1 |
| value: 72.521 |
| - type: ndcg_at_10 |
| value: 82.395 |
| - type: ndcg_at_100 |
| value: 83.554 |
| - type: ndcg_at_1000 |
| value: 83.774 |
| - type: ndcg_at_3 |
| value: 79.341 |
| - type: ndcg_at_5 |
| value: 81.036 |
| - type: precision_at_1 |
| value: 72.521 |
| - type: precision_at_10 |
| value: 9.812 |
| - type: precision_at_100 |
| value: 1.038 |
| - type: precision_at_1000 |
| value: 0.106 |
| - type: precision_at_3 |
| value: 29.694 |
| - type: precision_at_5 |
| value: 18.712999999999997 |
| - type: recall_at_1 |
| value: 70.244 |
| - type: recall_at_10 |
| value: 92.35 |
| - type: recall_at_100 |
| value: 97.419 |
| - type: recall_at_1000 |
| value: 99.16199999999999 |
| - type: recall_at_3 |
| value: 84.303 |
| - type: recall_at_5 |
| value: 88.325 |
| - type: main_score |
| value: 82.395 |
| task: |
| type: Retrieval |
| - dataset: |
| config: zh-CN |
| name: MTEB MassiveIntentClassification (zh-CN) |
| revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
| split: test |
| type: mteb/amazon_massive_intent |
| metrics: |
| - type: accuracy |
| value: 76.3752521856086 |
| - type: accuracy_stderr |
| value: 1.3911220977886072 |
| - type: f1 |
| value: 73.38330839246518 |
| - type: f1_stderr |
| value: 0.9864886479418102 |
| - type: main_score |
| value: 76.3752521856086 |
| task: |
| type: Classification |
| - dataset: |
| config: zh-CN |
| name: MTEB MassiveScenarioClassification (zh-CN) |
| revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
| split: test |
| type: mteb/amazon_massive_scenario |
| metrics: |
| - type: accuracy |
| value: 81.8022864828514 |
| - type: accuracy_stderr |
| value: 1.4060452754762354 |
| - type: f1 |
| value: 80.85164585310973 |
| - type: f1_stderr |
| value: 1.2664399398388577 |
| - type: main_score |
| value: 81.8022864828514 |
| task: |
| type: Classification |
| - dataset: |
| config: default |
| name: MTEB MedicalRetrieval |
| revision: None |
| split: dev |
| type: C-MTEB/MedicalRetrieval |
| metrics: |
| - type: map_at_1 |
| value: 57.199999999999996 |
| - type: map_at_10 |
| value: 63.346999999999994 |
| - type: map_at_100 |
| value: 63.852 |
| - type: map_at_1000 |
| value: 63.88700000000001 |
| - type: map_at_3 |
| value: 61.967000000000006 |
| - type: map_at_5 |
| value: 62.66199999999999 |
| - type: mrr_at_1 |
| value: 57.3 |
| - type: mrr_at_10 |
| value: 63.397000000000006 |
| - type: mrr_at_100 |
| value: 63.902 |
| - type: mrr_at_1000 |
| value: 63.937 |
| - type: mrr_at_3 |
| value: 62.017 |
| - type: mrr_at_5 |
| value: 62.712 |
| - type: ndcg_at_1 |
| value: 57.199999999999996 |
| - type: ndcg_at_10 |
| value: 66.38300000000001 |
| - type: ndcg_at_100 |
| value: 69.267 |
| - type: ndcg_at_1000 |
| value: 70.233 |
| - type: ndcg_at_3 |
| value: 63.44499999999999 |
| - type: ndcg_at_5 |
| value: 64.71000000000001 |
| - type: precision_at_1 |
| value: 57.199999999999996 |
| - type: precision_at_10 |
| value: 7.6 |
| - type: precision_at_100 |
| value: 0.905 |
| - type: precision_at_1000 |
| value: 0.098 |
| - type: precision_at_3 |
| value: 22.567 |
| - type: precision_at_5 |
| value: 14.16 |
| - type: recall_at_1 |
| value: 57.199999999999996 |
| - type: recall_at_10 |
| value: 76.0 |
| - type: recall_at_100 |
| value: 90.5 |
| - type: recall_at_1000 |
| value: 98.2 |
| - type: recall_at_3 |
| value: 67.7 |
| - type: recall_at_5 |
| value: 70.8 |
| - type: main_score |
| value: 66.38300000000001 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB MultilingualSentiment |
| revision: None |
| split: validation |
| type: C-MTEB/MultilingualSentiment-classification |
| metrics: |
| - type: accuracy |
| value: 80.12333333333335 |
| - type: accuracy_stderr |
| value: 0.31377628265303376 |
| - type: f1 |
| value: 80.26166732998303 |
| - type: f1_stderr |
| value: 0.2836457609943486 |
| - type: main_score |
| value: 80.12333333333335 |
| task: |
| type: Classification |
| - dataset: |
| config: default |
| name: MTEB Ocnli |
| revision: None |
| split: validation |
| type: C-MTEB/OCNLI |
| metrics: |
| - type: cos_sim_accuracy |
| value: 87.54737412019492 |
| - type: cos_sim_accuracy_threshold |
| value: 96.99121475650863 |
| - type: cos_sim_ap |
| value: 91.71816430648396 |
| - type: cos_sim_f1 |
| value: 88.27655310621243 |
| - type: cos_sim_f1_threshold |
| value: 96.8697507135398 |
| - type: cos_sim_precision |
| value: 83.98474737845567 |
| - type: cos_sim_recall |
| value: 93.03062302006336 |
| - type: dot_accuracy |
| value: 87.54737412019492 |
| - type: dot_accuracy_threshold |
| value: 96.99121475650863 |
| - type: dot_ap |
| value: 91.71816430648396 |
| - type: dot_f1 |
| value: 88.27655310621243 |
| - type: dot_f1_threshold |
| value: 96.86975071353979 |
| - type: dot_precision |
| value: 83.98474737845567 |
| - type: dot_recall |
| value: 93.03062302006336 |
| - type: euclidean_accuracy |
| value: 87.54737412019492 |
| - type: euclidean_accuracy_threshold |
| value: 24.530733065589622 |
| - type: euclidean_ap |
| value: 91.71816430648396 |
| - type: euclidean_f1 |
| value: 88.27655310621243 |
| - type: euclidean_f1_threshold |
| value: 25.020988098238107 |
| - type: euclidean_precision |
| value: 83.98474737845567 |
| - type: euclidean_recall |
| value: 93.03062302006336 |
| - type: manhattan_accuracy |
| value: 87.27666486193829 |
| - type: manhattan_accuracy_threshold |
| value: 752.4905438529156 |
| - type: manhattan_ap |
| value: 91.70647280240597 |
| - type: manhattan_f1 |
| value: 88.08920425747591 |
| - type: manhattan_f1_threshold |
| value: 752.4905438529156 |
| - type: manhattan_precision |
| value: 84.69785575048732 |
| - type: manhattan_recall |
| value: 91.76346356916578 |
| - type: max_accuracy |
| value: 87.54737412019492 |
| - type: max_ap |
| value: 91.71816430648396 |
| - type: max_f1 |
| value: 88.27655310621243 |
| task: |
| type: PairClassification |
| - dataset: |
| config: default |
| name: MTEB OnlineShopping |
| revision: None |
| split: test |
| type: C-MTEB/OnlineShopping-classification |
| metrics: |
| - type: accuracy |
| value: 94.46999999999998 |
| - type: accuracy_stderr |
| value: 0.2865309756378883 |
| - type: ap |
| value: 93.00417328431348 |
| - type: ap_stderr |
| value: 0.5383352662551945 |
| - type: f1 |
| value: 94.4618263222835 |
| - type: f1_stderr |
| value: 0.2840342094212124 |
| - type: main_score |
| value: 94.46999999999998 |
| task: |
| type: Classification |
| - dataset: |
| config: default |
| name: MTEB PAWSX |
| revision: None |
| split: test |
| type: C-MTEB/PAWSX |
| metrics: |
| - type: cosine_pearson |
| value: 46.85211982536296 |
| - type: cosine_spearman |
| value: 49.917839688145996 |
| - type: manhattan_pearson |
| value: 49.66820248148123 |
| - type: manhattan_spearman |
| value: 49.94013555794742 |
| - type: euclidean_pearson |
| value: 49.63608491973345 |
| - type: euclidean_spearman |
| value: 49.917839688145996 |
| - type: main_score |
| value: 49.917839688145996 |
| task: |
| type: STS |
| - dataset: |
| config: default |
| name: MTEB QBQTC |
| revision: None |
| split: test |
| type: C-MTEB/QBQTC |
| metrics: |
| - type: cosine_pearson |
| value: 55.18355221701257 |
| - type: cosine_spearman |
| value: 54.67390932826382 |
| - type: manhattan_pearson |
| value: 53.32847494683504 |
| - type: manhattan_spearman |
| value: 54.61660160532041 |
| - type: euclidean_pearson |
| value: 53.405599174765364 |
| - type: euclidean_spearman |
| value: 54.67390932826382 |
| - type: main_score |
| value: 54.67390932826382 |
| task: |
| type: STS |
| - dataset: |
| config: zh |
| name: MTEB STS22 (zh) |
| revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
| split: test |
| type: mteb/sts22-crosslingual-sts |
| metrics: |
| - type: cosine_pearson |
| value: 67.89319522460808 |
| - type: cosine_spearman |
| value: 68.98524514928238 |
| - type: manhattan_pearson |
| value: 67.65257700660463 |
| - type: manhattan_spearman |
| value: 69.17199742136434 |
| - type: euclidean_pearson |
| value: 67.52535570217756 |
| - type: euclidean_spearman |
| value: 68.98524514928238 |
| - type: main_score |
| value: 68.98524514928238 |
| task: |
| type: STS |
| - dataset: |
| config: default |
| name: MTEB STSB |
| revision: None |
| split: test |
| type: C-MTEB/STSB |
| metrics: |
| - type: cosine_pearson |
| value: 75.4871803618505 |
| - type: cosine_spearman |
| value: 76.17471665593993 |
| - type: manhattan_pearson |
| value: 75.73597640243183 |
| - type: manhattan_spearman |
| value: 76.20048941210949 |
| - type: euclidean_pearson |
| value: 75.66172628182565 |
| - type: euclidean_spearman |
| value: 76.17471665593993 |
| - type: main_score |
| value: 76.17471665593993 |
| task: |
| type: STS |
| - dataset: |
| config: default |
| name: MTEB T2Reranking |
| revision: None |
| split: dev |
| type: C-MTEB/T2Reranking |
| metrics: |
| - type: map |
| value: 67.45036855302303 |
| - type: mrr |
| value: 78.15107441080697 |
| - type: main_score |
| value: 67.45036855302303 |
| task: |
| type: Reranking |
| - dataset: |
| config: default |
| name: MTEB T2Retrieval |
| revision: None |
| split: dev |
| type: C-MTEB/T2Retrieval |
| metrics: |
| - type: map_at_1 |
| value: 28.094 |
| - type: map_at_10 |
| value: 79.367 |
| - type: map_at_100 |
| value: 82.89800000000001 |
| - type: map_at_1000 |
| value: 82.953 |
| - type: map_at_3 |
| value: 55.782 |
| - type: map_at_5 |
| value: 68.667 |
| - type: mrr_at_1 |
| value: 91.237 |
| - type: mrr_at_10 |
| value: 93.399 |
| - type: mrr_at_100 |
| value: 93.479 |
| - type: mrr_at_1000 |
| value: 93.482 |
| - type: mrr_at_3 |
| value: 93.029 |
| - type: mrr_at_5 |
| value: 93.273 |
| - type: ndcg_at_1 |
| value: 91.237 |
| - type: ndcg_at_10 |
| value: 86.368 |
| - type: ndcg_at_100 |
| value: 89.637 |
| - type: ndcg_at_1000 |
| value: 90.16300000000001 |
| - type: ndcg_at_3 |
| value: 87.691 |
| - type: ndcg_at_5 |
| value: 86.462 |
| - type: precision_at_1 |
| value: 91.237 |
| - type: precision_at_10 |
| value: 42.841 |
| - type: precision_at_100 |
| value: 5.047 |
| - type: precision_at_1000 |
| value: 0.517 |
| - type: precision_at_3 |
| value: 76.708 |
| - type: precision_at_5 |
| value: 64.428 |
| - type: recall_at_1 |
| value: 28.094 |
| - type: recall_at_10 |
| value: 85.181 |
| - type: recall_at_100 |
| value: 95.953 |
| - type: recall_at_1000 |
| value: 98.63 |
| - type: recall_at_3 |
| value: 57.267999999999994 |
| - type: recall_at_5 |
| value: 71.75399999999999 |
| - type: main_score |
| value: 86.368 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB TNews |
| revision: None |
| split: validation |
| type: C-MTEB/TNews-classification |
| metrics: |
| - type: accuracy |
| value: 55.482 |
| - type: accuracy_stderr |
| value: 0.3268577672321692 |
| - type: f1 |
| value: 53.57211848235611 |
| - type: f1_stderr |
| value: 0.3511138517262321 |
| - type: main_score |
| value: 55.482 |
| task: |
| type: Classification |
| - dataset: |
| config: default |
| name: MTEB ThuNewsClusteringP2P |
| revision: None |
| split: test |
| type: C-MTEB/ThuNewsClusteringP2P |
| metrics: |
| - type: v_measure |
| value: 79.44895384385426 |
| - type: v_measure_std |
| value: 2.315777338929376 |
| - type: main_score |
| value: 79.44895384385426 |
| task: |
| type: Clustering |
| - dataset: |
| config: default |
| name: MTEB ThuNewsClusteringS2S |
| revision: None |
| split: test |
| type: C-MTEB/ThuNewsClusteringS2S |
| metrics: |
| - type: v_measure |
| value: 76.95904984506356 |
| - type: v_measure_std |
| value: 2.244801218820472 |
| - type: main_score |
| value: 76.95904984506356 |
| task: |
| type: Clustering |
| - dataset: |
| config: default |
| name: MTEB VideoRetrieval |
| revision: None |
| split: dev |
| type: C-MTEB/VideoRetrieval |
| metrics: |
| - type: map_at_1 |
| value: 65.60000000000001 |
| - type: map_at_10 |
| value: 75.24499999999999 |
| - type: map_at_100 |
| value: 75.51 |
| - type: map_at_1000 |
| value: 75.519 |
| - type: map_at_3 |
| value: 73.68299999999999 |
| - type: map_at_5 |
| value: 74.638 |
| - type: mrr_at_1 |
| value: 65.60000000000001 |
| - type: mrr_at_10 |
| value: 75.24499999999999 |
| - type: mrr_at_100 |
| value: 75.51 |
| - type: mrr_at_1000 |
| value: 75.519 |
| - type: mrr_at_3 |
| value: 73.68299999999999 |
| - type: mrr_at_5 |
| value: 74.638 |
| - type: ndcg_at_1 |
| value: 65.60000000000001 |
| - type: ndcg_at_10 |
| value: 79.338 |
| - type: ndcg_at_100 |
| value: 80.585 |
| - type: ndcg_at_1000 |
| value: 80.772 |
| - type: ndcg_at_3 |
| value: 76.189 |
| - type: ndcg_at_5 |
| value: 77.915 |
| - type: precision_at_1 |
| value: 65.60000000000001 |
| - type: precision_at_10 |
| value: 9.19 |
| - type: precision_at_100 |
| value: 0.976 |
| - type: precision_at_1000 |
| value: 0.099 |
| - type: precision_at_3 |
| value: 27.800000000000004 |
| - type: precision_at_5 |
| value: 17.52 |
| - type: recall_at_1 |
| value: 65.60000000000001 |
| - type: recall_at_10 |
| value: 91.9 |
| - type: recall_at_100 |
| value: 97.6 |
| - type: recall_at_1000 |
| value: 99.0 |
| - type: recall_at_3 |
| value: 83.39999999999999 |
| - type: recall_at_5 |
| value: 87.6 |
| - type: main_score |
| value: 79.338 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB Waimai |
| revision: None |
| split: test |
| type: C-MTEB/waimai-classification |
| metrics: |
| - type: accuracy |
| value: 89.9 |
| - type: accuracy_stderr |
| value: 0.7861297602813425 |
| - type: ap |
| value: 76.33068327298966 |
| - type: ap_stderr |
| value: 1.6404446239337744 |
| - type: f1 |
| value: 88.66175970131309 |
| - type: f1_stderr |
| value: 0.7269675835542363 |
| - type: main_score |
| value: 89.9 |
| task: |
| type: Classification |
| tags: |
| - mteb |
| --- |
| # quark-llm-embedding-1.5B |
|
|
| - Chinese Text Embedding Model developed by Alibaba Quark-LLM Team. Details will be published later. |
