Add indicators for each evals

src/schema.py

@@ -1,6 +1,5 @@
 from dataclasses import dataclass, make_dataclass
 from enum import Enum
-from typing import Literal, Optional, Union
 
 from src.display.formatting import make_clickable_model
 
@@ -66,10 +65,10 @@ model_registry = {
 
 ### The Task and Tasks classes store information about each benchmark we're scoring.  ###
 class TaskCategory(Enum):
-    CULTURAL_KNOWLEDGE = "Cultural Knowledge"
-    CLASSICAL_NLP = "Classical NLP"
-    READING_COMPREHENSION = "Reading Comprehension"
-    TRANSLATION = "Translation"
+    CULTURAL_KNOWLEDGE = "🌏 Cultural Knowledge"
+    CLASSICAL_NLP = "🏛️ Classical NLP"
+    READING_COMPREHENSION = "📖 Reading Comprehension"
+    TRANSLATION = "🔢 Generation"
 
 
 @dataclass
@@ -84,28 +83,28 @@ class Task:
 
 class Tasks(Enum):
     # fmt: off
-    balita_tgl_mcf = Task("balita_tgl_mcf", "acc_", "BalitaNLP", "tgl", TaskCategory.CULTURAL_KNOWLEDGE, 35_177)
-    belebele_ceb_mcf = Task("belebele_ceb_mcf", "acc_", "Belebele (ceb)", "ceb", TaskCategory.READING_COMPREHENSION, 900)
-    belebele_fil_mcf = Task("belebele_fil_mcf", "acc_", "Belebele (fil)", "fil", TaskCategory.READING_COMPREHENSION, 900)
-    cebuaner_ceb_mcf = Task("cebuaner_ceb_mcf", "acc_", "CebuaNER", "ceb", TaskCategory.CLASSICAL_NLP, 1310)
-    dengue_filipino_fil = Task("dengue_filipino_fil:_average", "acc_norm", "Dengue", "fil", TaskCategory.CLASSICAL_NLP, 4015)
-    firecs_fil_mcf = Task("firecs_fil_mcf", "acc_", "FiReCS", "fil", TaskCategory.CLASSICAL_NLP, 7340)
-    global_mmlu_all_tgl = Task("global_mmlu_all_tgl_mcf:_average", "acc_", "Global-MMLU", "tgl", TaskCategory.CULTURAL_KNOWLEDGE, 14_042)
-    include_tgl_mcf = Task("include_tgl_mcf:_average", "acc_", "INCLUDE", "tgl", TaskCategory.CULTURAL_KNOWLEDGE, 500)
-    kalahi_tgl_mcf = Task("kalahi_tgl_mcf", "acc_", "KALAHI", "tgl", TaskCategory.CULTURAL_KNOWLEDGE, 150)
-    newsphnli_fil_mcf = Task("newsphnli_fil_mcf", "acc_", "NewsPH NLI", "fil", TaskCategory.READING_COMPREHENSION, 90_000)
-    ntrex128_fil = Task("ntrex128_fil", "bleu", "NTREX-128", "fil", TaskCategory.TRANSLATION, 1997)
-    readability_ceb_mcf = Task("readability_ceb_mcf", "acc_", "Readability (ceb)", "ceb", TaskCategory.READING_COMPREHENSION, 350)
-    sib200_ceb_mcf = Task("sib200_ceb_mcf", "acc_", "SIB-200 (ceb)", "ceb", TaskCategory.CLASSICAL_NLP, 99)
-    sib200_tgl_mcf = Task("sib200_tgl_mcf", "acc_", "SIB-200 (tgl)", "tgl", TaskCategory.CLASSICAL_NLP, 99)
+    balita_tgl_mcf = Task("balita_tgl_mcf", "acc_", "🌏 BalitaNLP", "tgl", TaskCategory.CULTURAL_KNOWLEDGE, 35_177)
+    belebele_ceb_mcf = Task("belebele_ceb_mcf", "acc_", "📖 Belebele (ceb)", "ceb", TaskCategory.READING_COMPREHENSION, 900)
+    belebele_fil_mcf = Task("belebele_fil_mcf", "acc_", "📖 Belebele (fil)", "fil", TaskCategory.READING_COMPREHENSION, 900)
+    cebuaner_ceb_mcf = Task("cebuaner_ceb_mcf", "acc_", "🏛️ CebuaNER", "ceb", TaskCategory.CLASSICAL_NLP, 1310)
+    dengue_filipino_fil = Task("dengue_filipino_fil:_average", "acc_norm", "🏛️ Dengue", "fil", TaskCategory.CLASSICAL_NLP, 4015)
+    firecs_fil_mcf = Task("firecs_fil_mcf", "acc_", "🏛️ FiReCS", "fil", TaskCategory.CLASSICAL_NLP, 7340)
+    global_mmlu_all_tgl = Task("global_mmlu_all_tgl_mcf:_average", "acc_", "🌏 Global-MMLU", "tgl", TaskCategory.CULTURAL_KNOWLEDGE, 14_042)
+    include_tgl_mcf = Task("include_tgl_mcf:_average", "acc_", "🌏 INCLUDE", "tgl", TaskCategory.CULTURAL_KNOWLEDGE, 500)
+    kalahi_tgl_mcf = Task("kalahi_tgl_mcf", "acc_", "🌏 KALAHI", "tgl", TaskCategory.CULTURAL_KNOWLEDGE, 150)
+    newsphnli_fil_mcf = Task("newsphnli_fil_mcf", "acc_", "📖 NewsPH NLI", "fil", TaskCategory.READING_COMPREHENSION, 90_000)
+    ntrex128_fil = Task("ntrex128_fil", "bleu", "🔢 NTREX-128", "fil", TaskCategory.TRANSLATION, 1997)
+    readability_ceb_mcf = Task("readability_ceb_mcf", "acc_", "📖 Readability (ceb)", "ceb", TaskCategory.READING_COMPREHENSION, 350)
+    sib200_ceb_mcf = Task("sib200_ceb_mcf", "acc_", "🏛️ SIB-200 (ceb)", "ceb", TaskCategory.CLASSICAL_NLP, 99)
+    sib200_tgl_mcf = Task("sib200_tgl_mcf", "acc_", "🏛️ SIB-200 (tgl)", "tgl", TaskCategory.CLASSICAL_NLP, 99)
     # stingraybench_corr_tgl_mcf = Task("stingraybench_correctness_tgl_mcf", "acc_", "StingrayBench (Correctness)", "tgl", TaskCategory.CULTURAL_KNOWLEDGE, 100)
-    stingraybench_sem_appropriateness_tgl_mcf = Task("stingraybench_semantic_appropriateness_tgl_mcf", "acc_", "StingrayBench", "tgl", TaskCategory.CULTURAL_KNOWLEDGE, 100)
-    tatoeba_ceb = Task("tatoeba_ceb", "bleu", "Tatoeba (ceb)", "ceb", TaskCategory.TRANSLATION, 377)
-    tatoeba_tgl = Task("tatoeba_tgl", "bleu", "Tatoeba (tgl)", "tgl", TaskCategory.TRANSLATION, 2499)
-    tico19_tgl = Task("tico19_tgl", "bleu", "TICO-19", "tgl", TaskCategory.TRANSLATION, 971)
-    tlunifiedner_tgl_mcf = Task("tlunifiedner_tgl_mcf", "acc_", "TLUnified NER", "tgl", TaskCategory.CLASSICAL_NLP, 1579)
-    universalner_ceb_mcf = Task("universalner_ceb_mcf", "acc_", "Universal NER (ceb)", "ceb", TaskCategory.CLASSICAL_NLP, 49)
-    universalner_tgl_mcf = Task("universalner_tgl_mcf", "acc_", "Universal NER (tgl)", "tgl", TaskCategory.CLASSICAL_NLP, 56)
+    stingraybench_sem_appropriateness_tgl_mcf = Task("stingraybench_semantic_appropriateness_tgl_mcf", "acc_", "🌏StingrayBench", "tgl", TaskCategory.CULTURAL_KNOWLEDGE, 100)
+    tatoeba_ceb = Task("tatoeba_ceb", "bleu", "🔢 Tatoeba (ceb)", "ceb", TaskCategory.TRANSLATION, 377)
+    tatoeba_tgl = Task("tatoeba_tgl", "bleu", "🔢 Tatoeba (tgl)", "tgl", TaskCategory.TRANSLATION, 2499)
+    tico19_tgl = Task("tico19_tgl", "bleu", "🔢 TICO-19", "tgl", TaskCategory.TRANSLATION, 971)
+    tlunifiedner_tgl_mcf = Task("tlunifiedner_tgl_mcf", "acc_", "🏛️ TLUnified NER", "tgl", TaskCategory.CLASSICAL_NLP, 1579)
+    universalner_ceb_mcf = Task("universalner_ceb_mcf", "acc_", "🏛️ Universal NER (ceb)", "ceb", TaskCategory.CLASSICAL_NLP, 49)
+    universalner_tgl_mcf = Task("universalner_tgl_mcf", "acc_", "🏛️ Universal NER (tgl)", "tgl", TaskCategory.CLASSICAL_NLP, 56)
     # fmt: on