{"meta":{"generated":"2026-07-30","corpora":["flores200","massive"],"n_sentences":{"flores":997,"massive":5000},"pivot":"eng_Latn","note":"Fertility is the median per-sentence ratio against English on the same tokenizer. Floor is the lowest median token count any tokenizer in this registry achieves for the language; neglect is the excess above that floor, measured within each corpus."},"tokenizers":[{"key":"gpt2","family":"Historical baseline","vocab":50257,"note":"GPT-2 byte-level BPE. Shows how bad an English-only vocabulary was."},{"key":"cl100k","family":"OpenAI legacy","vocab":100277,"note":"Used by GPT-3.5 and GPT-4 era models."},{"key":"o200k","family":"OpenAI current","vocab":200019,"note":"Current OpenAI encoding."},{"key":"llama3","family":"Llama","vocab":128256,"note":"Llama-3 tokenizer."},{"key":"gemma","family":"Gemma / Gemini","vocab":256000,"note":"Gemma SentencePiece vocabulary."},{"key":"tekken","family":"Mistral","vocab":131072,"note":"Tekken, read from the repo's HF-format tokenizer.json."},{"key":"qwen2","family":"Qwen","vocab":151646,"note":"Qwen2 tokenizer."},{"key":"bloom","family":"Multilingual-first","vocab":250680,"note":"Vocabulary allocated multilingually from the start. The ceiling."}],"languages":[{"code":"ace_Arab","name":"Acehnese","script":"Arabic","family":"Austronesian","region":"Eurasia;Papunesia","speakers":3500032,"floor":60,"floors":{"flores":60},"metrics":{"flores":{"gpt2":{"tokens":116,"fertility":4.74,"p90":5.81,"cpt":0.87,"neglect":0.93},"cl100k":{"tokens":92,"fertility":3.74,"p90":4.7,"cpt":1.09,"neglect":0.53},"o200k":{"tokens":64,"fertility":2.57,"p90":3.28,"cpt":1.6,"neglect":0.07},"llama3":{"tokens":70,"fertility":2.82,"p90":3.65,"cpt":1.44,"neglect":0.17},"gemma":{"tokens":60,"fertility":2.39,"p90":3.05,"cpt":1.7,"neglect":0},"tekken":{"tokens":63,"fertility":2.45,"p90":3.14,"cpt":1.62,"neglect":0.05},"qwen2":{"tokens":66,"fertility":2.63,"p90":3.27,"cpt":1.53,"neglect":0.1},"bloom":{"tokens":65,"fertility":2.59,"p90":3.35,"cpt":1.56,"neglect":0.08}}}},{"code":"ace_Latn","name":"Acehnese","script":"Latin","family":"Austronesian","region":"Eurasia;Papunesia","speakers":3500032,"floor":43,"floors":{"flores":43},"metrics":{"flores":{"gpt2":{"tokens":53,"fertility":2.14,"p90":2.73,"cpt":2.43,"neglect":0.23},"cl100k":{"tokens":49,"fertility":1.97,"p90":2.48,"cpt":2.64,"neglect":0.14},"o200k":{"tokens":44,"fertility":1.78,"p90":2.27,"cpt":2.93,"neglect":0.02},"llama3":{"tokens":49,"fertility":1.94,"p90":2.47,"cpt":2.67,"neglect":0.14},"gemma":{"tokens":44,"fertility":1.77,"p90":2.26,"cpt":2.93,"neglect":0.02},"tekken":{"tokens":47,"fertility":1.84,"p90":2.35,"cpt":2.75,"neglect":0.09},"qwen2":{"tokens":49,"fertility":1.94,"p90":2.47,"cpt":2.63,"neglect":0.14},"bloom":{"tokens":43,"fertility":1.74,"p90":2.19,"cpt":2.98,"neglect":0}}}},{"code":"acm_Arab","name":"Gilit Mesopotamian Arabic","script":"Arabic","family":"Afro-Asiatic","region":"Eurasia","speakers":15700000,"floor":29,"floors":{"flores":29},"metrics":{"flores":{"gpt2":{"tokens":105,"fertility":4.17,"p90":5.23,"cpt":0.99,"neglect":2.62},"cl100k":{"tokens":74,"fertility":2.96,"p90":3.74,"cpt":1.41,"neglect":1.55},"o200k":{"tokens":34,"fertility":1.38,"p90":1.69,"cpt":3.05,"neglect":0.17},"llama3":{"tokens":41,"fertility":1.64,"p90":2.02,"cpt":2.54,"neglect":0.41},"gemma":{"tokens":37,"fertility":1.5,"p90":1.86,"cpt":2.79,"neglect":0.28},"tekken":{"tokens":32,"fertility":1.26,"p90":1.56,"cpt":3.22,"neglect":0.1},"qwen2":{"tokens":41,"fertility":1.61,"p90":2,"cpt":2.53,"neglect":0.41},"bloom":{"tokens":29,"fertility":1.15,"p90":1.42,"cpt":3.59,"neglect":0}}}},{"code":"acq_Arab","name":"Ta'izzi-Adeni Arabic","script":"Arabic","family":"Afro-Asiatic","region":"Eurasia","speakers":10500000,"floor":29,"floors":{"flores":29},"metrics":{"flores":{"gpt2":{"tokens":106,"fertility":4.26,"p90":5.29,"cpt":0.99,"neglect":2.66},"cl100k":{"tokens":75,"fertility":3,"p90":3.7,"cpt":1.41,"neglect":1.59},"o200k":{"tokens":34,"fertility":1.38,"p90":1.69,"cpt":3.08,"neglect":0.17},"llama3":{"tokens":41,"fertility":1.65,"p90":2.04,"cpt":2.56,"neglect":0.41},"gemma":{"tokens":37,"fertility":1.49,"p90":1.86,"cpt":2.83,"neglect":0.28},"tekken":{"tokens":32,"fertility":1.26,"p90":1.53,"cpt":3.27,"neglect":0.1},"qwen2":{"tokens":41,"fertility":1.62,"p90":2,"cpt":2.56,"neglect":0.41},"bloom":{"tokens":29,"fertility":1.15,"p90":1.42,"cpt":3.62,"neglect":0}}}},{"code":"aeb_Arab","name":"Tunisian Arabic","script":"Arabic","family":"Afro-Asiatic","region":"Africa;Eurasia","speakers":11600000,"floor":30,"floors":{"flores":30},"metrics":{"flores":{"gpt2":{"tokens":103,"fertility":4.16,"p90":5.2,"cpt":0.99,"neglect":2.43},"cl100k":{"tokens":73,"fertility":2.91,"p90":3.67,"cpt":1.42,"neglect":1.43},"o200k":{"tokens":35,"fertility":1.42,"p90":1.8,"cpt":2.93,"neglect":0.17},"llama3":{"tokens":41,"fertility":1.65,"p90":2.08,"cpt":2.49,"neglect":0.37},"gemma":{"tokens":38,"fertility":1.52,"p90":1.93,"cpt":2.71,"neglect":0.27},"tekken":{"tokens":33,"fertility":1.3,"p90":1.65,"cpt":3.08,"neglect":0.1},"qwen2":{"tokens":42,"fertility":1.65,"p90":2.08,"cpt":2.45,"neglect":0.4},"bloom":{"tokens":30,"fertility":1.2,"p90":1.5,"cpt":3.41,"neglect":0}}}},{"code":"afr_Latn","name":"Afrikaans","script":"Latin","family":"Indo-European","region":"Africa","speakers":10300000,"floor":35,"floors":{"flores":35,"massive":9},"metrics":{"flores":{"gpt2":{"tokens":48,"fertility":1.93,"p90":2.33,"cpt":2.67,"neglect":0.37},"cl100k":{"tokens":41,"fertility":1.69,"p90":2.04,"cpt":3.05,"neglect":0.17},"o200k":{"tokens":35,"fertility":1.42,"p90":1.69,"cpt":3.65,"neglect":0},"llama3":{"tokens":41,"fertility":1.68,"p90":2.03,"cpt":3.06,"neglect":0.17},"gemma":{"tokens":36,"fertility":1.46,"p90":1.72,"cpt":3.52,"neglect":0.03},"tekken":{"tokens":39,"fertility":1.52,"p90":1.83,"cpt":3.29,"neglect":0.11},"qwen2":{"tokens":42,"fertility":1.68,"p90":2.04,"cpt":3.02,"neglect":0.2},"bloom":{"tokens":42,"fertility":1.68,"p90":2.03,"cpt":3.03,"neglect":0.2}},"massive":{"gpt2":{"tokens":12,"fertility":1.8,"p90":2.5,"cpt":2.72,"neglect":0.33},"cl100k":{"tokens":11,"fertility":1.64,"p90":2.25,"cpt":3.08,"neglect":0.22},"o200k":{"tokens":9,"fertility":1.38,"p90":1.9,"cpt":3.71,"neglect":0},"llama3":{"tokens":11,"fertility":1.62,"p90":2.25,"cpt":3.09,"neglect":0.22},"gemma":{"tokens":10,"fertility":1.43,"p90":2,"cpt":3.58,"neglect":0.11},"tekken":{"tokens":10,"fertility":1.5,"p90":2,"cpt":3.26,"neglect":0.11},"qwen2":{"tokens":11,"fertility":1.62,"p90":2.25,"cpt":3.08,"neglect":0.22},"bloom":{"tokens":11,"fertility":1.6,"p90":2.2,"cpt":3.06,"neglect":0.22}}}},{"code":"ajp_Arab","name":"South Levantine Arabic","script":"Arabic","family":"Afro-Asiatic","region":"Eurasia","speakers":11000000,"floor":28,"floors":{"flores":28},"metrics":{"flores":{"gpt2":{"tokens":99,"fertility":3.95,"p90":4.91,"cpt":1.01,"neglect":2.54},"cl100k":{"tokens":70,"fertility":2.81,"p90":3.49,"cpt":1.42,"neglect":1.5},"o200k":{"tokens":33,"fertility":1.32,"p90":1.63,"cpt":3.02,"neglect":0.18},"llama3":{"tokens":39,"fertility":1.56,"p90":1.94,"cpt":2.54,"neglect":0.39},"gemma":{"tokens":36,"fertility":1.42,"p90":1.74,"cpt":2.81,"neglect":0.29},"tekken":{"tokens":32,"fertility":1.22,"p90":1.5,"cpt":3.17,"neglect":0.14},"qwen2":{"tokens":39,"fertility":1.53,"p90":1.92,"cpt":2.54,"neglect":0.39},"bloom":{"tokens":28,"fertility":1.11,"p90":1.36,"cpt":3.54,"neglect":0}}}},{"code":"aka_Latn","name":"Akan","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":11000000,"floor":51,"floors":{"flores":51},"metrics":{"flores":{"gpt2":{"tokens":68,"fertility":2.72,"p90":3.67,"cpt":1.76,"neglect":0.33},"cl100k":{"tokens":65,"fertility":2.62,"p90":3.54,"cpt":1.83,"neglect":0.27},"o200k":{"tokens":51,"fertility":2.05,"p90":2.72,"cpt":2.35,"neglect":0},"llama3":{"tokens":65,"fertility":2.61,"p90":3.52,"cpt":1.83,"neglect":0.27},"gemma":{"tokens":54,"fertility":2.18,"p90":2.93,"cpt":2.19,"neglect":0.06},"tekken":{"tokens":64,"fertility":2.5,"p90":3.4,"cpt":1.85,"neglect":0.25},"qwen2":{"tokens":57,"fertility":2.22,"p90":2.94,"cpt":2.12,"neglect":0.12},"bloom":{"tokens":51,"fertility":2,"p90":2.67,"cpt":2.35,"neglect":0}}}},{"code":"als_Latn","name":"Tosk Albanian","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":1800000,"floor":45,"floors":{"flores":45,"massive":11},"metrics":{"flores":{"gpt2":{"tokens":66,"fertility":2.67,"p90":3.37,"cpt":2.06,"neglect":0.47},"cl100k":{"tokens":56,"fertility":2.25,"p90":2.86,"cpt":2.42,"neglect":0.24},"o200k":{"tokens":45,"fertility":1.82,"p90":2.24,"cpt":3.01,"neglect":0},"llama3":{"tokens":55,"fertility":2.23,"p90":2.82,"cpt":2.45,"neglect":0.22},"gemma":{"tokens":50,"fertility":2.04,"p90":2.55,"cpt":2.69,"neglect":0.11},"tekken":{"tokens":47,"fertility":1.88,"p90":2.33,"cpt":2.84,"neglect":0.04},"qwen2":{"tokens":56,"fertility":2.24,"p90":2.84,"cpt":2.41,"neglect":0.24},"bloom":{"tokens":54,"fertility":2.18,"p90":2.71,"cpt":2.49,"neglect":0.2}},"massive":{"gpt2":{"tokens":17,"fertility":2.43,"p90":3.5,"cpt":2.04,"neglect":0.55},"cl100k":{"tokens":14,"fertility":2,"p90":3,"cpt":2.45,"neglect":0.27},"o200k":{"tokens":11,"fertility":1.67,"p90":2.5,"cpt":3.05,"neglect":0},"llama3":{"tokens":14,"fertility":2,"p90":3,"cpt":2.48,"neglect":0.27},"gemma":{"tokens":12,"fertility":1.83,"p90":2.67,"cpt":2.79,"neglect":0.09},"tekken":{"tokens":12,"fertility":1.71,"p90":2.5,"cpt":2.88,"neglect":0.09},"qwen2":{"tokens":14,"fertility":2,"p90":3,"cpt":2.45,"neglect":0.27},"bloom":{"tokens":14,"fertility":2,"p90":2.8,"cpt":2.51,"neglect":0.27}}}},{"code":"amh_Ethi","name":"Amharic","script":"Ethiopic","family":"Afro-Asiatic","region":"Africa","speakers":21900000,"floor":80,"floors":{"flores":80,"massive":22},"metrics":{"flores":{"gpt2":{"tokens":195,"fertility":7.82,"p90":10,"cpt":0.42,"neglect":1.44},"cl100k":{"tokens":194,"fertility":7.75,"p90":9.92,"cpt":0.43,"neglect":1.43},"o200k":{"tokens":146,"fertility":5.83,"p90":7.51,"cpt":0.57,"neglect":0.82},"llama3":{"tokens":194,"fertility":7.74,"p90":9.91,"cpt":0.43,"neglect":1.43},"gemma":{"tokens":80,"fertility":3.2,"p90":4.09,"cpt":1.04,"neglect":0},"tekken":{"tokens":194,"fertility":7.59,"p90":9.81,"cpt":0.43,"neglect":1.43},"qwen2":{"tokens":105,"fertility":4.13,"p90":5.32,"cpt":0.79,"neglect":0.31},"bloom":{"tokens":129,"fertility":5.07,"p90":6.46,"cpt":0.64,"neglect":0.61}},"massive":{"gpt2":{"tokens":54,"fertility":7.7,"p90":11,"cpt":0.42,"neglect":1.45},"cl100k":{"tokens":53,"fertility":7.8,"p90":11.23,"cpt":0.42,"neglect":1.41},"o200k":{"tokens":40,"fertility":5.9,"p90":8.4,"cpt":0.56,"neglect":0.82},"llama3":{"tokens":53,"fertility":7.75,"p90":11.2,"cpt":0.42,"neglect":1.41},"gemma":{"tokens":22,"fertility":3.25,"p90":4.6,"cpt":1.02,"neglect":0},"tekken":{"tokens":53,"fertility":7.5,"p90":10.83,"cpt":0.42,"neglect":1.41},"qwen2":{"tokens":28,"fertility":4.14,"p90":5.83,"cpt":0.79,"neglect":0.27},"bloom":{"tokens":35,"fertility":5,"p90":7.17,"cpt":0.64,"neglect":0.59}}}},{"code":"apc_Arab","name":"Levantine Arabic","script":"Arabic","family":"Afro-Asiatic","region":"Eurasia","speakers":44000000,"floor":28,"floors":{"flores":28},"metrics":{"flores":{"gpt2":{"tokens":99,"fertility":4,"p90":5.02,"cpt":1,"neglect":2.54},"cl100k":{"tokens":70,"fertility":2.81,"p90":3.53,"cpt":1.41,"neglect":1.5},"o200k":{"tokens":33,"fertility":1.35,"p90":1.65,"cpt":2.98,"neglect":0.18},"llama3":{"tokens":40,"fertility":1.58,"p90":1.97,"cpt":2.5,"neglect":0.43},"gemma":{"tokens":36,"fertility":1.45,"p90":1.8,"cpt":2.75,"neglect":0.29},"tekken":{"tokens":31,"fertility":1.23,"p90":1.52,"cpt":3.13,"neglect":0.11},"qwen2":{"tokens":39,"fertility":1.57,"p90":1.95,"cpt":2.48,"neglect":0.39},"bloom":{"tokens":28,"fertility":1.14,"p90":1.41,"cpt":3.48,"neglect":0}}}},{"code":"arb_Arab","name":"Modern Standard Arabic","script":"Arabic","family":"Afro-Asiatic","region":"Eurasia","speakers":274000000,"floor":29,"floors":{"flores":29,"massive":7},"metrics":{"flores":{"gpt2":{"tokens":109,"fertility":4.32,"p90":5.4,"cpt":0.98,"neglect":2.76},"cl100k":{"tokens":76,"fertility":3,"p90":3.77,"cpt":1.41,"neglect":1.62},"o200k":{"tokens":34,"fertility":1.38,"p90":1.69,"cpt":3.13,"neglect":0.17},"llama3":{"tokens":42,"fertility":1.65,"p90":2.03,"cpt":2.58,"neglect":0.45},"gemma":{"tokens":37,"fertility":1.49,"p90":1.86,"cpt":2.87,"neglect":0.28},"tekken":{"tokens":32,"fertility":1.24,"p90":1.52,"cpt":3.34,"neglect":0.1},"qwen2":{"tokens":41,"fertility":1.62,"p90":2.05,"cpt":2.58,"neglect":0.41},"bloom":{"tokens":29,"fertility":1.14,"p90":1.42,"cpt":3.68,"neglect":0}},"massive":{"gpt2":{"tokens":25,"fertility":3.67,"p90":5.56,"cpt":1.01,"neglect":2.57},"cl100k":{"tokens":17,"fertility":2.67,"p90":4,"cpt":1.43,"neglect":1.43},"o200k":{"tokens":8,"fertility":1.22,"p90":1.88,"cpt":3.13,"neglect":0.14},"llama3":{"tokens":9,"fertility":1.43,"p90":2.22,"cpt":2.63,"neglect":0.29},"gemma":{"tokens":9,"fertility":1.33,"p90":2,"cpt":2.9,"neglect":0.29},"tekken":{"tokens":8,"fertility":1.17,"p90":1.75,"cpt":3.18,"neglect":0.14},"qwen2":{"tokens":9,"fertility":1.4,"p90":2.17,"cpt":2.68,"neglect":0.29},"bloom":{"tokens":7,"fertility":1,"p90":1.5,"cpt":3.62,"neglect":0}}}},{"code":"arb_Latn","name":"Standard Arabic","script":"Latin","family":"Afro-Asiatic","region":"Eurasia","speakers":335000000,"floor":54,"floors":{"flores":54},"metrics":{"flores":{"gpt2":{"tokens":62,"fertility":2.48,"p90":3.2,"cpt":2.27,"neglect":0.15},"cl100k":{"tokens":60,"fertility":2.43,"p90":3.15,"cpt":2.32,"neglect":0.11},"o200k":{"tokens":55,"fertility":2.23,"p90":2.86,"cpt":2.56,"neglect":0.02},"llama3":{"tokens":59,"fertility":2.4,"p90":3.12,"cpt":2.34,"neglect":0.09},"gemma":{"tokens":56,"fertility":2.28,"p90":2.95,"cpt":2.48,"neglect":0.04},"tekken":{"tokens":59,"fertility":2.31,"p90":2.98,"cpt":2.39,"neglect":0.09},"qwen2":{"tokens":61,"fertility":2.41,"p90":3.12,"cpt":2.31,"neglect":0.13},"bloom":{"tokens":54,"fertility":2.12,"p90":2.71,"cpt":2.63,"neglect":0}}}},{"code":"ars_Arab","name":"Najdi Arabic","script":"Arabic","family":"Afro-Asiatic","region":"Eurasia","speakers":10000000,"floor":29,"floors":{"flores":29},"metrics":{"flores":{"gpt2":{"tokens":109,"fertility":4.33,"p90":5.42,"cpt":0.98,"neglect":2.76},"cl100k":{"tokens":76,"fertility":3,"p90":3.77,"cpt":1.41,"neglect":1.62},"o200k":{"tokens":34,"fertility":1.38,"p90":1.7,"cpt":3.12,"neglect":0.17},"llama3":{"tokens":42,"fertility":1.65,"p90":2.04,"cpt":2.58,"neglect":0.45},"gemma":{"tokens":37,"fertility":1.49,"p90":1.86,"cpt":2.86,"neglect":0.28},"tekken":{"tokens":32,"fertility":1.24,"p90":1.53,"cpt":3.33,"neglect":0.1},"qwen2":{"tokens":41,"fertility":1.62,"p90":2.05,"cpt":2.58,"neglect":0.41},"bloom":{"tokens":29,"fertility":1.15,"p90":1.43,"cpt":3.67,"neglect":0}}}},{"code":"ary_Arab","name":"Moroccan Arabic","script":"Arabic","family":"Afro-Asiatic","region":"Africa","speakers":27500000,"floor":32,"floors":{"flores":32},"metrics":{"flores":{"gpt2":{"tokens":102,"fertility":4.13,"p90":5.32,"cpt":1,"neglect":2.19},"cl100k":{"tokens":73,"fertility":2.92,"p90":3.77,"cpt":1.42,"neglect":1.28},"o200k":{"tokens":36,"fertility":1.46,"p90":1.89,"cpt":2.84,"neglect":0.12},"llama3":{"tokens":42,"fertility":1.69,"p90":2.18,"cpt":2.45,"neglect":0.31},"gemma":{"tokens":38,"fertility":1.54,"p90":2,"cpt":2.67,"neglect":0.19},"tekken":{"tokens":35,"fertility":1.35,"p90":1.75,"cpt":2.97,"neglect":0.09},"qwen2":{"tokens":43,"fertility":1.66,"p90":2.15,"cpt":2.42,"neglect":0.34},"bloom":{"tokens":32,"fertility":1.26,"p90":1.61,"cpt":3.25,"neglect":0}}}},{"code":"arz_Arab","name":"Egyptian Arabic","script":"Arabic","family":"Afro-Asiatic","region":"Africa;Eurasia","speakers":64600000,"floor":30,"floors":{"flores":30},"metrics":{"flores":{"gpt2":{"tokens":104,"fertility":4.17,"p90":5.33,"cpt":1,"neglect":2.47},"cl100k":{"tokens":73,"fertility":2.92,"p90":3.72,"cpt":1.42,"neglect":1.43},"o200k":{"tokens":35,"fertility":1.4,"p90":1.75,"cpt":3,"neglect":0.17},"llama3":{"tokens":41,"fertility":1.64,"p90":2.06,"cpt":2.54,"neglect":0.37},"gemma":{"tokens":37,"fertility":1.48,"p90":1.87,"cpt":2.8,"neglect":0.23},"tekken":{"tokens":33,"fertility":1.28,"p90":1.62,"cpt":3.16,"neglect":0.1},"qwen2":{"tokens":41,"fertility":1.61,"p90":2.06,"cpt":2.52,"neglect":0.37},"bloom":{"tokens":30,"fertility":1.17,"p90":1.47,"cpt":3.53,"neglect":0}}}},{"code":"asm_Beng","name":"Assamese","script":"Bengali","family":"Indo-European","region":"Eurasia","speakers":15300000,"floor":35,"floors":{"flores":35},"metrics":{"flores":{"gpt2":{"tokens":242,"fertility":9.7,"p90":12.22,"cpt":0.48,"neglect":5.91},"cl100k":{"tokens":155,"fertility":6.19,"p90":7.8,"cpt":0.76,"neglect":3.43},"o200k":{"tokens":49,"fertility":2,"p90":2.48,"cpt":2.37,"neglect":0.4},"llama3":{"tokens":154,"fertility":6.17,"p90":7.76,"cpt":0.76,"neglect":3.4},"gemma":{"tokens":81,"fertility":3.25,"p90":4.13,"cpt":1.45,"neglect":1.31},"tekken":{"tokens":85,"fertility":3.32,"p90":4.22,"cpt":1.38,"neglect":1.43},"qwen2":{"tokens":138,"fertility":5.47,"p90":6.94,"cpt":0.85,"neglect":2.94},"bloom":{"tokens":35,"fertility":1.42,"p90":1.74,"cpt":3.27,"neglect":0}}}},{"code":"ast_Latn","name":"Asturian-Leonese-Cantabrian","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":450000,"floor":32,"floors":{"flores":32},"metrics":{"flores":{"gpt2":{"tokens":47,"fertility":1.86,"p90":2.32,"cpt":2.67,"neglect":0.47},"cl100k":{"tokens":39,"fertility":1.56,"p90":1.94,"cpt":3.17,"neglect":0.22},"o200k":{"tokens":34,"fertility":1.4,"p90":1.72,"cpt":3.58,"neglect":0.06},"llama3":{"tokens":38,"fertility":1.55,"p90":1.92,"cpt":3.21,"neglect":0.19},"gemma":{"tokens":34,"fertility":1.38,"p90":1.71,"cpt":3.59,"neglect":0.06},"tekken":{"tokens":35,"fertility":1.39,"p90":1.72,"cpt":3.48,"neglect":0.09},"qwen2":{"tokens":39,"fertility":1.54,"p90":1.92,"cpt":3.16,"neglect":0.22},"bloom":{"tokens":32,"fertility":1.29,"p90":1.59,"cpt":3.79,"neglect":0}}}},{"code":"awa_Deva","name":"Awadhi","script":"Devanagari","family":"Indo-European","region":"Eurasia","speakers":22000000,"floor":36,"floors":{"flores":36},"metrics":{"flores":{"gpt2":{"tokens":177,"fertility":7.17,"p90":9,"cpt":0.66,"neglect":3.92},"cl100k":{"tokens":119,"fertility":4.79,"p90":6,"cpt":1,"neglect":2.31},"o200k":{"tokens":42,"fertility":1.69,"p90":2.12,"cpt":2.81,"neglect":0.17},"llama3":{"tokens":63,"fertility":2.53,"p90":3.18,"cpt":1.87,"neglect":0.75},"gemma":{"tokens":51,"fertility":2.03,"p90":2.55,"cpt":2.33,"neglect":0.42},"tekken":{"tokens":52,"fertility":2.03,"p90":2.55,"cpt":2.27,"neglect":0.44},"qwen2":{"tokens":110,"fertility":4.35,"p90":5.5,"cpt":1.08,"neglect":2.06},"bloom":{"tokens":36,"fertility":1.42,"p90":1.78,"cpt":3.28,"neglect":0}}}},{"code":"ayr_Latn","name":"Central Aymara","script":"Latin","family":"Aymaran","region":"South America","speakers":1700000,"floor":46,"floors":{"flores":46},"metrics":{"flores":{"gpt2":{"tokens":56,"fertility":2.3,"p90":3.06,"cpt":2.21,"neglect":0.22},"cl100k":{"tokens":53,"fertility":2.16,"p90":2.85,"cpt":2.35,"neglect":0.15},"o200k":{"tokens":46,"fertility":1.9,"p90":2.55,"cpt":2.66,"neglect":0},"llama3":{"tokens":52,"fertility":2.14,"p90":2.79,"cpt":2.37,"neglect":0.13},"gemma":{"tokens":48,"fertility":1.95,"p90":2.64,"cpt":2.58,"neglect":0.04},"tekken":{"tokens":50,"fertility":2,"p90":2.64,"cpt":2.46,"neglect":0.09},"qwen2":{"tokens":53,"fertility":2.13,"p90":2.82,"cpt":2.33,"neglect":0.15},"bloom":{"tokens":47,"fertility":1.92,"p90":2.54,"cpt":2.61,"neglect":0.02}}}},{"code":"azb_Arab","name":"South Azerbaijani","script":"Arabic","family":"Turkic","region":"Eurasia","speakers":13800000,"floor":44,"floors":{"flores":44},"metrics":{"flores":{"gpt2":{"tokens":127,"fertility":5.11,"p90":6.56,"cpt":0.85,"neglect":1.89},"cl100k":{"tokens":82,"fertility":3.32,"p90":4.27,"cpt":1.29,"neglect":0.86},"o200k":{"tokens":46,"fertility":1.85,"p90":2.37,"cpt":2.34,"neglect":0.05},"llama3":{"tokens":47,"fertility":1.9,"p90":2.46,"cpt":2.25,"neglect":0.07},"gemma":{"tokens":44,"fertility":1.78,"p90":2.29,"cpt":2.4,"neglect":0},"tekken":{"tokens":45,"fertility":1.76,"p90":2.25,"cpt":2.37,"neglect":0.02},"qwen2":{"tokens":66,"fertility":2.6,"p90":3.38,"cpt":1.64,"neglect":0.5},"bloom":{"tokens":45,"fertility":1.8,"p90":2.29,"cpt":2.35,"neglect":0.02}}}},{"code":"azj_Latn","name":"North Azerbaijani","script":"Latin","family":"Turkic","region":"Eurasia","speakers":9200000,"floor":42,"floors":{"flores":42,"massive":11},"metrics":{"flores":{"gpt2":{"tokens":86,"fertility":3.43,"p90":4.71,"cpt":1.54,"neglect":1.05},"cl100k":{"tokens":66,"fertility":2.62,"p90":3.56,"cpt":2.01,"neglect":0.57},"o200k":{"tokens":42,"fertility":1.69,"p90":2.19,"cpt":3.13,"neglect":0},"llama3":{"tokens":60,"fertility":2.39,"p90":3.29,"cpt":2.19,"neglect":0.43},"gemma":{"tokens":52,"fertility":2.06,"p90":2.79,"cpt":2.54,"neglect":0.24},"tekken":{"tokens":46,"fertility":1.79,"p90":2.36,"cpt":2.85,"neglect":0.1},"qwen2":{"tokens":63,"fertility":2.45,"p90":3.38,"cpt":2.1,"neglect":0.5},"bloom":{"tokens":58,"fertility":2.27,"p90":3.04,"cpt":2.29,"neglect":0.38}},"massive":{"gpt2":{"tokens":21,"fertility":3,"p90":4.67,"cpt":1.55,"neglect":0.91},"cl100k":{"tokens":16,"fertility":2.36,"p90":3.57,"cpt":2.05,"neglect":0.45},"o200k":{"tokens":11,"fertility":1.64,"p90":2.5,"cpt":3.02,"neglect":0},"llama3":{"tokens":15,"fertility":2.2,"p90":3.33,"cpt":2.22,"neglect":0.36},"gemma":{"tokens":13,"fertility":1.91,"p90":2.89,"cpt":2.59,"neglect":0.18},"tekken":{"tokens":12,"fertility":1.75,"p90":2.67,"cpt":2.71,"neglect":0.09},"qwen2":{"tokens":15,"fertility":2.25,"p90":3.43,"cpt":2.16,"neglect":0.36},"bloom":{"tokens":14,"fertility":2,"p90":3,"cpt":2.32,"neglect":0.27}}}},{"code":"bak_Cyrl","name":"Bashkir","script":"Cyrillic","family":"Turkic","region":"Eurasia","speakers":1200000,"floor":43,"floors":{"flores":43},"metrics":{"flores":{"gpt2":{"tokens":147,"fertility":5.93,"p90":7.65,"cpt":0.83,"neglect":2.42},"cl100k":{"tokens":106,"fertility":4.24,"p90":5.58,"cpt":1.16,"neglect":1.47},"o200k":{"tokens":43,"fertility":1.74,"p90":2.18,"cpt":2.84,"neglect":0},"llama3":{"tokens":91,"fertility":3.65,"p90":4.86,"cpt":1.34,"neglect":1.12},"gemma":{"tokens":62,"fertility":2.52,"p90":3.27,"cpt":1.96,"neglect":0.44},"tekken":{"tokens":66,"fertility":2.59,"p90":3.37,"cpt":1.86,"neglect":0.53},"qwen2":{"tokens":78,"fertility":3.12,"p90":4.06,"cpt":1.55,"neglect":0.81},"bloom":{"tokens":89,"fertility":3.55,"p90":4.5,"cpt":1.37,"neglect":1.07}}}},{"code":"bam_Latn","name":"Bambara","script":"Latin","family":"Mande","region":"Africa","speakers":4200000,"floor":47,"floors":{"flores":47},"metrics":{"flores":{"gpt2":{"tokens":65,"fertility":2.63,"p90":3.5,"cpt":1.76,"neglect":0.38},"cl100k":{"tokens":63,"fertility":2.55,"p90":3.37,"cpt":1.82,"neglect":0.34},"o200k":{"tokens":50,"fertility":2.03,"p90":2.63,"cpt":2.31,"neglect":0.06},"llama3":{"tokens":62,"fertility":2.5,"p90":3.33,"cpt":1.85,"neglect":0.32},"gemma":{"tokens":51,"fertility":2.08,"p90":2.68,"cpt":2.25,"neglect":0.09},"tekken":{"tokens":62,"fertility":2.43,"p90":3.25,"cpt":1.86,"neglect":0.32},"qwen2":{"tokens":54,"fertility":2.12,"p90":2.76,"cpt":2.15,"neglect":0.15},"bloom":{"tokens":47,"fertility":1.87,"p90":2.43,"cpt":2.45,"neglect":0}}}},{"code":"ban_Latn","name":"Balinese","script":"Latin","family":"Austronesian","region":"Papunesia","speakers":4000000,"floor":38,"floors":{"flores":38},"metrics":{"flores":{"gpt2":{"tokens":50,"fertility":1.97,"p90":2.55,"cpt":2.74,"neglect":0.32},"cl100k":{"tokens":46,"fertility":1.81,"p90":2.32,"cpt":2.98,"neglect":0.21},"o200k":{"tokens":39,"fertility":1.57,"p90":2.03,"cpt":3.44,"neglect":0.03},"llama3":{"tokens":45,"fertility":1.8,"p90":2.29,"cpt":3.01,"neglect":0.18},"gemma":{"tokens":39,"fertility":1.55,"p90":2,"cpt":3.49,"neglect":0.03},"tekken":{"tokens":43,"fertility":1.67,"p90":2.15,"cpt":3.15,"neglect":0.13},"qwen2":{"tokens":46,"fertility":1.8,"p90":2.32,"cpt":2.96,"neglect":0.21},"bloom":{"tokens":38,"fertility":1.48,"p90":1.91,"cpt":3.63,"neglect":0}}}},{"code":"bel_Cyrl","name":"Belarusian","script":"Cyrillic","family":"Indo-European","region":"Eurasia","speakers":7600000,"floor":50,"floors":{"flores":50},"metrics":{"flores":{"gpt2":{"tokens":164,"fertility":6.45,"p90":8.79,"cpt":0.85,"neglect":2.28},"cl100k":{"tokens":89,"fertility":3.5,"p90":4.73,"cpt":1.56,"neglect":0.78},"o200k":{"tokens":50,"fertility":1.98,"p90":2.62,"cpt":2.78,"neglect":0},"llama3":{"tokens":66,"fertility":2.59,"p90":3.48,"cpt":2.12,"neglect":0.32},"gemma":{"tokens":58,"fertility":2.29,"p90":3.09,"cpt":2.38,"neglect":0.16},"tekken":{"tokens":56,"fertility":2.16,"p90":2.87,"cpt":2.48,"neglect":0.12},"qwen2":{"tokens":77,"fertility":2.95,"p90":4.1,"cpt":1.81,"neglect":0.54},"bloom":{"tokens":82,"fertility":3.2,"p90":4.31,"cpt":1.69,"neglect":0.64}}}},{"code":"bem_Latn","name":"Bemba (Zambia)","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":3600000,"floor":48,"floors":{"flores":48},"metrics":{"flores":{"gpt2":{"tokens":62,"fertility":2.43,"p90":3.2,"cpt":2.44,"neglect":0.29},"cl100k":{"tokens":56,"fertility":2.21,"p90":2.87,"cpt":2.68,"neglect":0.17},"o200k":{"tokens":48,"fertility":1.93,"p90":2.49,"cpt":3.1,"neglect":0},"llama3":{"tokens":56,"fertility":2.19,"p90":2.86,"cpt":2.7,"neglect":0.17},"gemma":{"tokens":51,"fertility":2.02,"p90":2.62,"cpt":2.94,"neglect":0.06},"tekken":{"tokens":55,"fertility":2.1,"p90":2.72,"cpt":2.75,"neglect":0.15},"qwen2":{"tokens":57,"fertility":2.19,"p90":2.84,"cpt":2.66,"neglect":0.19},"bloom":{"tokens":49,"fertility":1.9,"p90":2.43,"cpt":3.09,"neglect":0.02}}}},{"code":"ben_Beng","name":"Bengali","script":"Bengali","family":"Indo-European","region":"Eurasia","speakers":300000000,"floor":29,"floors":{"flores":29,"massive":8},"metrics":{"flores":{"gpt2":{"tokens":239,"fertility":9.57,"p90":12.17,"cpt":0.5,"neglect":7.24},"cl100k":{"tokens":145,"fertility":5.81,"p90":7.39,"cpt":0.82,"neglect":4},"o200k":{"tokens":42,"fertility":1.69,"p90":2.12,"cpt":2.82,"neglect":0.45},"llama3":{"tokens":145,"fertility":5.76,"p90":7.36,"cpt":0.82,"neglect":4},"gemma":{"tokens":67,"fertility":2.7,"p90":3.39,"cpt":1.77,"neglect":1.31},"tekken":{"tokens":53,"fertility":2.04,"p90":2.58,"cpt":2.26,"neglect":0.83},"qwen2":{"tokens":128,"fertility":5.03,"p90":6.46,"cpt":0.93,"neglect":3.41},"bloom":{"tokens":29,"fertility":1.18,"p90":1.46,"cpt":4.02,"neglect":0}},"massive":{"gpt2":{"tokens":62,"fertility":9,"p90":13.2,"cpt":0.5,"neglect":6.75},"cl100k":{"tokens":38,"fertility":5.71,"p90":8.25,"cpt":0.82,"neglect":3.75},"o200k":{"tokens":11,"fertility":1.67,"p90":2.5,"cpt":2.87,"neglect":0.38},"llama3":{"tokens":38,"fertility":5.71,"p90":8.25,"cpt":0.82,"neglect":3.75},"gemma":{"tokens":17,"fertility":2.6,"p90":3.8,"cpt":1.85,"neglect":1.12},"tekken":{"tokens":14,"fertility":2.1,"p90":3,"cpt":2.17,"neglect":0.75},"qwen2":{"tokens":33,"fertility":5,"p90":7.14,"cpt":0.95,"neglect":3.12},"bloom":{"tokens":8,"fertility":1.17,"p90":1.75,"cpt":3.93,"neglect":0}}}},{"code":"bho_Deva","name":"Bhojpuri","script":"Devanagari","family":"Indo-European","region":"Eurasia","speakers":52200000,"floor":38,"floors":{"flores":38},"metrics":{"flores":{"gpt2":{"tokens":176,"fertility":7.06,"p90":9.14,"cpt":0.66,"neglect":3.63},"cl100k":{"tokens":116,"fertility":4.63,"p90":5.93,"cpt":1.01,"neglect":2.05},"o200k":{"tokens":43,"fertility":1.75,"p90":2.21,"cpt":2.7,"neglect":0.13},"llama3":{"tokens":64,"fertility":2.56,"p90":3.28,"cpt":1.83,"neglect":0.68},"gemma":{"tokens":52,"fertility":2.08,"p90":2.67,"cpt":2.25,"neglect":0.37},"tekken":{"tokens":53,"fertility":2.05,"p90":2.62,"cpt":2.22,"neglect":0.39},"qwen2":{"tokens":108,"fertility":4.28,"p90":5.49,"cpt":1.08,"neglect":1.84},"bloom":{"tokens":38,"fertility":1.52,"p90":1.92,"cpt":3.06,"neglect":0}}}},{"code":"bjn_Arab","name":"Banjar","script":"Arabic","family":"Austronesian","region":"Papunesia","speakers":3500000,"floor":58,"floors":{"flores":58},"metrics":{"flores":{"gpt2":{"tokens":123,"fertility":4.96,"p90":6.07,"cpt":0.91,"neglect":1.12},"cl100k":{"tokens":94,"fertility":3.78,"p90":4.66,"cpt":1.19,"neglect":0.62},"o200k":{"tokens":63,"fertility":2.54,"p90":3.13,"cpt":1.79,"neglect":0.09},"llama3":{"tokens":67,"fertility":2.71,"p90":3.39,"cpt":1.65,"neglect":0.16},"gemma":{"tokens":58,"fertility":2.32,"p90":2.89,"cpt":1.94,"neglect":0},"tekken":{"tokens":62,"fertility":2.44,"p90":3.04,"cpt":1.8,"neglect":0.07},"qwen2":{"tokens":66,"fertility":2.59,"p90":3.21,"cpt":1.7,"neglect":0.14},"bloom":{"tokens":62,"fertility":2.46,"p90":3.05,"cpt":1.81,"neglect":0.07}}}},{"code":"bjn_Latn","name":"Banjar","script":"Latin","family":"Austronesian","region":"Papunesia","speakers":3500000,"floor":33,"floors":{"flores":33},"metrics":{"flores":{"gpt2":{"tokens":49,"fertility":2,"p90":2.46,"cpt":2.59,"neglect":0.48},"cl100k":{"tokens":43,"fertility":1.71,"p90":2.12,"cpt":2.99,"neglect":0.3},"o200k":{"tokens":36,"fertility":1.44,"p90":1.75,"cpt":3.58,"neglect":0.09},"llama3":{"tokens":42,"fertility":1.71,"p90":2.11,"cpt":3,"neglect":0.27},"gemma":{"tokens":35,"fertility":1.39,"p90":1.72,"cpt":3.68,"neglect":0.06},"tekken":{"tokens":40,"fertility":1.56,"p90":1.93,"cpt":3.21,"neglect":0.21},"qwen2":{"tokens":43,"fertility":1.71,"p90":2.11,"cpt":2.96,"neglect":0.3},"bloom":{"tokens":33,"fertility":1.31,"p90":1.6,"cpt":3.88,"neglect":0}}}},{"code":"bod_Tibt","name":"Tibetan","script":"Tibetan","family":"Sino-Tibetan","region":"Eurasia","speakers":1200000,"floor":89,"floors":{"flores":89},"metrics":{"flores":{"gpt2":{"tokens":365,"fertility":14.84,"p90":18.88,"cpt":0.37,"neglect":3.1},"cl100k":{"tokens":277,"fertility":11.24,"p90":14.35,"cpt":0.49,"neglect":2.11},"o200k":{"tokens":199,"fertility":8.11,"p90":10.43,"cpt":0.68,"neglect":1.24},"llama3":{"tokens":277,"fertility":11.24,"p90":14.36,"cpt":0.49,"neglect":2.11},"gemma":{"tokens":89,"fertility":3.65,"p90":4.63,"cpt":1.51,"neglect":0},"tekken":{"tokens":312,"fertility":12.42,"p90":15.87,"cpt":0.43,"neglect":2.51},"qwen2":{"tokens":181,"fertility":7.27,"p90":9.42,"cpt":0.74,"neglect":1.03},"bloom":{"tokens":165,"fertility":6.64,"p90":8.42,"cpt":0.82,"neglect":0.85}}}},{"code":"bos_Latn","name":"Bosnian","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":2700000,"floor":36,"floors":{"flores":36},"metrics":{"flores":{"gpt2":{"tokens":54,"fertility":2.18,"p90":2.74,"cpt":2.25,"neglect":0.5},"cl100k":{"tokens":46,"fertility":1.87,"p90":2.33,"cpt":2.61,"neglect":0.28},"o200k":{"tokens":36,"fertility":1.49,"p90":1.83,"cpt":3.31,"neglect":0},"llama3":{"tokens":45,"fertility":1.78,"p90":2.25,"cpt":2.72,"neglect":0.25},"gemma":{"tokens":40,"fertility":1.61,"p90":2,"cpt":3.02,"neglect":0.11},"tekken":{"tokens":38,"fertility":1.5,"p90":1.88,"cpt":3.14,"neglect":0.06},"qwen2":{"tokens":47,"fertility":1.86,"p90":2.33,"cpt":2.59,"neglect":0.31},"bloom":{"tokens":46,"fertility":1.83,"p90":2.27,"cpt":2.64,"neglect":0.28}}}},{"code":"bug_Latn","name":"Buginese","script":"Latin","family":"Austronesian","region":"Papunesia","speakers":5017800,"floor":43,"floors":{"flores":43},"metrics":{"flores":{"gpt2":{"tokens":55,"fertility":2.21,"p90":2.89,"cpt":2.35,"neglect":0.28},"cl100k":{"tokens":50,"fertility":2,"p90":2.59,"cpt":2.6,"neglect":0.16},"o200k":{"tokens":43,"fertility":1.75,"p90":2.28,"cpt":2.97,"neglect":0},"llama3":{"tokens":49,"fertility":1.96,"p90":2.56,"cpt":2.63,"neglect":0.14},"gemma":{"tokens":43,"fertility":1.74,"p90":2.3,"cpt":2.96,"neglect":0},"tekken":{"tokens":47,"fertility":1.82,"p90":2.36,"cpt":2.77,"neglect":0.09},"qwen2":{"tokens":50,"fertility":1.97,"p90":2.58,"cpt":2.58,"neglect":0.16},"bloom":{"tokens":43,"fertility":1.73,"p90":2.25,"cpt":2.99,"neglect":0}}}},{"code":"bul_Cyrl","name":"Bulgarian","script":"Cyrillic","family":"Indo-European","region":"Eurasia","speakers":9000000,"floor":40,"floors":{"flores":40},"metrics":{"flores":{"gpt2":{"tokens":136,"fertility":5.5,"p90":6.91,"cpt":0.93,"neglect":2.4},"cl100k":{"tokens":65,"fertility":2.64,"p90":3.27,"cpt":1.93,"neglect":0.62},"o200k":{"tokens":42,"fertility":1.71,"p90":2.1,"cpt":2.99,"neglect":0.05},"llama3":{"tokens":51,"fertility":2.05,"p90":2.51,"cpt":2.49,"neglect":0.28},"gemma":{"tokens":40,"fertility":1.62,"p90":2,"cpt":3.12,"neglect":0},"tekken":{"tokens":42,"fertility":1.64,"p90":2,"cpt":3,"neglect":0.05},"qwen2":{"tokens":55,"fertility":2.2,"p90":2.75,"cpt":2.26,"neglect":0.38},"bloom":{"tokens":62,"fertility":2.47,"p90":3.07,"cpt":2.03,"neglect":0.55}}}},{"code":"cat_Latn","name":"Catalan","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":4873396,"floor":29,"floors":{"flores":29,"massive":8},"metrics":{"flores":{"gpt2":{"tokens":47,"fertility":1.9,"p90":2.37,"cpt":2.81,"neglect":0.62},"cl100k":{"tokens":42,"fertility":1.71,"p90":2.1,"cpt":3.13,"neglect":0.45},"o200k":{"tokens":36,"fertility":1.47,"p90":1.78,"cpt":3.66,"neglect":0.24},"llama3":{"tokens":42,"fertility":1.69,"p90":2.09,"cpt":3.15,"neglect":0.45},"gemma":{"tokens":37,"fertility":1.52,"p90":1.88,"cpt":3.52,"neglect":0.28},"tekken":{"tokens":36,"fertility":1.44,"p90":1.74,"cpt":3.64,"neglect":0.24},"qwen2":{"tokens":42,"fertility":1.68,"p90":2.08,"cpt":3.12,"neglect":0.45},"bloom":{"tokens":29,"fertility":1.18,"p90":1.39,"cpt":4.51,"neglect":0}},"massive":{"gpt2":{"tokens":13,"fertility":1.86,"p90":2.75,"cpt":2.76,"neglect":0.62},"cl100k":{"tokens":11,"fertility":1.67,"p90":2.5,"cpt":3.17,"neglect":0.38},"o200k":{"tokens":10,"fertility":1.45,"p90":2.2,"cpt":3.64,"neglect":0.25},"llama3":{"tokens":11,"fertility":1.67,"p90":2.5,"cpt":3.19,"neglect":0.38},"gemma":{"tokens":10,"fertility":1.5,"p90":2.23,"cpt":3.54,"neglect":0.25},"tekken":{"tokens":10,"fertility":1.44,"p90":2.14,"cpt":3.52,"neglect":0.25},"qwen2":{"tokens":11,"fertility":1.67,"p90":2.5,"cpt":3.18,"neglect":0.38},"bloom":{"tokens":8,"fertility":1.17,"p90":1.67,"cpt":4.39,"neglect":0}}}},{"code":"ceb_Latn","name":"Cebuano","script":"Latin","family":"Austronesian","region":"Papunesia","speakers":15900000,"floor":41,"floors":{"flores":41},"metrics":{"flores":{"gpt2":{"tokens":55,"fertility":2.23,"p90":2.8,"cpt":2.63,"neglect":0.34},"cl100k":{"tokens":47,"fertility":1.92,"p90":2.38,"cpt":3.03,"neglect":0.15},"o200k":{"tokens":41,"fertility":1.67,"p90":2.05,"cpt":3.53,"neglect":0},"llama3":{"tokens":47,"fertility":1.91,"p90":2.33,"cpt":3.06,"neglect":0.15},"gemma":{"tokens":43,"fertility":1.77,"p90":2.19,"cpt":3.31,"neglect":0.05},"tekken":{"tokens":46,"fertility":1.84,"p90":2.25,"cpt":3.11,"neglect":0.12},"qwen2":{"tokens":48,"fertility":1.92,"p90":2.37,"cpt":3,"neglect":0.17},"bloom":{"tokens":44,"fertility":1.77,"p90":2.17,"cpt":3.27,"neglect":0.07}}}},{"code":"ces_Latn","name":"Czech","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":10700000,"floor":37,"floors":{"flores":37},"metrics":{"flores":{"gpt2":{"tokens":65,"fertility":2.61,"p90":3.36,"cpt":1.81,"neglect":0.76},"cl100k":{"tokens":52,"fertility":2.11,"p90":2.67,"cpt":2.24,"neglect":0.41},"o200k":{"tokens":39,"fertility":1.59,"p90":1.96,"cpt":2.99,"neglect":0.05},"llama3":{"tokens":37,"fertility":1.5,"p90":1.85,"cpt":3.14,"neglect":0},"gemma":{"tokens":37,"fertility":1.49,"p90":1.85,"cpt":3.15,"neglect":0},"tekken":{"tokens":39,"fertility":1.55,"p90":1.91,"cpt":2.97,"neglect":0.05},"qwen2":{"tokens":52,"fertility":2.08,"p90":2.64,"cpt":2.24,"neglect":0.41},"bloom":{"tokens":51,"fertility":2.03,"p90":2.55,"cpt":2.31,"neglect":0.38}}}},{"code":"cjk_Latn","name":"Chokwe","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":1000000,"floor":43,"floors":{"flores":43},"metrics":{"flores":{"gpt2":{"tokens":54,"fertility":2.17,"p90":2.91,"cpt":2.42,"neglect":0.26},"cl100k":{"tokens":49,"fertility":1.98,"p90":2.65,"cpt":2.64,"neglect":0.14},"o200k":{"tokens":43,"fertility":1.75,"p90":2.35,"cpt":3.01,"neglect":0},"llama3":{"tokens":49,"fertility":1.97,"p90":2.63,"cpt":2.66,"neglect":0.14},"gemma":{"tokens":44,"fertility":1.77,"p90":2.39,"cpt":2.94,"neglect":0.02},"tekken":{"tokens":48,"fertility":1.89,"p90":2.57,"cpt":2.69,"neglect":0.12},"qwen2":{"tokens":49,"fertility":1.94,"p90":2.59,"cpt":2.65,"neglect":0.14},"bloom":{"tokens":43,"fertility":1.73,"p90":2.33,"cpt":3.01,"neglect":0}}}},{"code":"ckb_Arab","name":"Central Kurdish","script":"Arabic","family":"Indo-European","region":"Eurasia","speakers":7250000,"floor":67,"floors":{"flores":67},"metrics":{"flores":{"gpt2":{"tokens":160,"fertility":6.39,"p90":8.13,"cpt":0.73,"neglect":1.39},"cl100k":{"tokens":119,"fertility":4.77,"p90":6.08,"cpt":0.99,"neglect":0.78},"o200k":{"tokens":67,"fertility":2.74,"p90":3.49,"cpt":1.72,"neglect":0},"llama3":{"tokens":98,"fertility":3.91,"p90":5.1,"cpt":1.19,"neglect":0.46},"gemma":{"tokens":70,"fertility":2.83,"p90":3.67,"cpt":1.66,"neglect":0.04},"tekken":{"tokens":96,"fertility":3.78,"p90":4.96,"cpt":1.21,"neglect":0.43},"qwen2":{"tokens":87,"fertility":3.42,"p90":4.4,"cpt":1.34,"neglect":0.3},"bloom":{"tokens":80,"fertility":3.17,"p90":4.09,"cpt":1.46,"neglect":0.19}}}},{"code":"crh_Latn","name":"Crimean Tatar","script":"Latin","family":"Turkic","region":"Eurasia","speakers":552740,"floor":42,"floors":{"flores":42},"metrics":{"flores":{"gpt2":{"tokens":62,"fertility":2.47,"p90":3.15,"cpt":2.02,"neglect":0.48},"cl100k":{"tokens":53,"fertility":2.1,"p90":2.67,"cpt":2.36,"neglect":0.26},"o200k":{"tokens":42,"fertility":1.7,"p90":2.14,"cpt":2.94,"neglect":0},"llama3":{"tokens":46,"fertility":1.82,"p90":2.31,"cpt":2.73,"neglect":0.1},"gemma":{"tokens":44,"fertility":1.75,"p90":2.25,"cpt":2.84,"neglect":0.05},"tekken":{"tokens":46,"fertility":1.77,"p90":2.27,"cpt":2.72,"neglect":0.1},"qwen2":{"tokens":50,"fertility":1.94,"p90":2.48,"cpt":2.52,"neglect":0.19},"bloom":{"tokens":52,"fertility":2.06,"p90":2.61,"cpt":2.39,"neglect":0.24}}}},{"code":"cym_Latn","name":"Welsh","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":724000,"floor":42,"floors":{"flores":42,"massive":11},"metrics":{"flores":{"gpt2":{"tokens":58,"fertility":2.35,"p90":2.93,"cpt":2.22,"neglect":0.38},"cl100k":{"tokens":53,"fertility":2.12,"p90":2.65,"cpt":2.45,"neglect":0.26},"o200k":{"tokens":42,"fertility":1.73,"p90":2.08,"cpt":3.04,"neglect":0},"llama3":{"tokens":53,"fertility":2.11,"p90":2.63,"cpt":2.46,"neglect":0.26},"gemma":{"tokens":49,"fertility":1.97,"p90":2.45,"cpt":2.64,"neglect":0.17},"tekken":{"tokens":51,"fertility":2,"p90":2.45,"cpt":2.55,"neglect":0.21},"qwen2":{"tokens":53,"fertility":2.1,"p90":2.64,"cpt":2.44,"neglect":0.26},"bloom":{"tokens":53,"fertility":2.1,"p90":2.59,"cpt":2.46,"neglect":0.26}},"massive":{"gpt2":{"tokens":16,"fertility":2.33,"p90":3.4,"cpt":2.2,"neglect":0.45},"cl100k":{"tokens":14,"fertility":2.11,"p90":3,"cpt":2.49,"neglect":0.27},"o200k":{"tokens":11,"fertility":1.71,"p90":2.5,"cpt":3.09,"neglect":0},"llama3":{"tokens":14,"fertility":2.08,"p90":3,"cpt":2.51,"neglect":0.27},"gemma":{"tokens":13,"fertility":2,"p90":2.83,"cpt":2.73,"neglect":0.18},"tekken":{"tokens":14,"fertility":2,"p90":2.83,"cpt":2.59,"neglect":0.27},"qwen2":{"tokens":14,"fertility":2.1,"p90":3,"cpt":2.5,"neglect":0.27},"bloom":{"tokens":14,"fertility":2,"p90":3,"cpt":2.51,"neglect":0.27}}}},{"code":"dan_Latn","name":"Danish","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":6000000,"floor":34,"floors":{"flores":34,"massive":8},"metrics":{"flores":{"gpt2":{"tokens":47,"fertility":1.87,"p90":2.35,"cpt":2.66,"neglect":0.38},"cl100k":{"tokens":40,"fertility":1.61,"p90":1.97,"cpt":3.1,"neglect":0.18},"o200k":{"tokens":34,"fertility":1.38,"p90":1.67,"cpt":3.65,"neglect":0},"llama3":{"tokens":40,"fertility":1.6,"p90":1.96,"cpt":3.11,"neglect":0.18},"gemma":{"tokens":34,"fertility":1.36,"p90":1.67,"cpt":3.64,"neglect":0},"tekken":{"tokens":37,"fertility":1.47,"p90":1.79,"cpt":3.32,"neglect":0.09},"qwen2":{"tokens":40,"fertility":1.6,"p90":1.97,"cpt":3.07,"neglect":0.18},"bloom":{"tokens":42,"fertility":1.67,"p90":2.05,"cpt":2.97,"neglect":0.24}},"massive":{"gpt2":{"tokens":12,"fertility":1.69,"p90":2.38,"cpt":2.7,"neglect":0.5},"cl100k":{"tokens":10,"fertility":1.44,"p90":2,"cpt":3.23,"neglect":0.25},"o200k":{"tokens":9,"fertility":1.25,"p90":1.75,"cpt":3.74,"neglect":0.12},"llama3":{"tokens":10,"fertility":1.43,"p90":2,"cpt":3.24,"neglect":0.25},"gemma":{"tokens":8,"fertility":1.25,"p90":1.75,"cpt":3.79,"neglect":0},"tekken":{"tokens":9,"fertility":1.33,"p90":1.86,"cpt":3.37,"neglect":0.12},"qwen2":{"tokens":10,"fertility":1.44,"p90":2,"cpt":3.23,"neglect":0.25},"bloom":{"tokens":11,"fertility":1.5,"p90":2,"cpt":3.02,"neglect":0.38}}}},{"code":"deu_Latn","name":"German","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":76540740,"floor":31,"floors":{"flores":31,"massive":8},"metrics":{"flores":{"gpt2":{"tokens":53,"fertility":2.14,"p90":2.71,"cpt":2.66,"neglect":0.71},"cl100k":{"tokens":39,"fertility":1.56,"p90":1.93,"cpt":3.61,"neglect":0.26},"o200k":{"tokens":32,"fertility":1.3,"p90":1.55,"cpt":4.41,"neglect":0.03},"llama3":{"tokens":39,"fertility":1.56,"p90":1.92,"cpt":3.62,"neglect":0.26},"gemma":{"tokens":31,"fertility":1.24,"p90":1.5,"cpt":4.54,"neglect":0},"tekken":{"tokens":33,"fertility":1.3,"p90":1.58,"cpt":4.25,"neglect":0.06},"qwen2":{"tokens":39,"fertility":1.54,"p90":1.89,"cpt":3.6,"neglect":0.26},"bloom":{"tokens":42,"fertility":1.68,"p90":2.05,"cpt":3.36,"neglect":0.35}},"massive":{"gpt2":{"tokens":13,"fertility":2,"p90":2.71,"cpt":2.71,"neglect":0.62},"cl100k":{"tokens":10,"fertility":1.43,"p90":2,"cpt":3.73,"neglect":0.25},"o200k":{"tokens":8,"fertility":1.25,"p90":1.75,"cpt":4.31,"neglect":0},"llama3":{"tokens":10,"fertility":1.43,"p90":2,"cpt":3.74,"neglect":0.25},"gemma":{"tokens":8,"fertility":1.18,"p90":1.67,"cpt":4.56,"neglect":0},"tekken":{"tokens":9,"fertility":1.29,"p90":1.78,"cpt":4.05,"neglect":0.12},"qwen2":{"tokens":10,"fertility":1.4,"p90":2,"cpt":3.77,"neglect":0.25},"bloom":{"tokens":11,"fertility":1.5,"p90":2.17,"cpt":3.43,"neglect":0.38}}}},{"code":"dik_Latn","name":"Southwestern Dinka","script":"Latin","family":"Nilotic","region":"Africa","speakers":1365900,"floor":44,"floors":{"flores":44},"metrics":{"flores":{"gpt2":{"tokens":59,"fertility":2.33,"p90":3.72,"cpt":1.7,"neglect":0.34},"cl100k":{"tokens":54,"fertility":2.14,"p90":3.32,"cpt":1.86,"neglect":0.23},"o200k":{"tokens":45,"fertility":1.79,"p90":2.68,"cpt":2.26,"neglect":0.02},"llama3":{"tokens":52,"fertility":2.09,"p90":3.21,"cpt":1.91,"neglect":0.18},"gemma":{"tokens":45,"fertility":1.77,"p90":2.63,"cpt":2.26,"neglect":0.02},"tekken":{"tokens":51,"fertility":1.97,"p90":2.95,"cpt":1.97,"neglect":0.16},"qwen2":{"tokens":49,"fertility":1.93,"p90":2.9,"cpt":2.04,"neglect":0.11},"bloom":{"tokens":44,"fertility":1.74,"p90":2.64,"cpt":2.28,"neglect":0}}}},{"code":"dyu_Latn","name":"Dyula","script":"Latin","family":"Mande","region":"Africa","speakers":2700000,"floor":45,"floors":{"flores":45},"metrics":{"flores":{"gpt2":{"tokens":54,"fertility":2.16,"p90":2.93,"cpt":2.25,"neglect":0.2},"cl100k":{"tokens":50,"fertility":2.04,"p90":2.71,"cpt":2.4,"neglect":0.11},"o200k":{"tokens":45,"fertility":1.83,"p90":2.45,"cpt":2.68,"neglect":0},"llama3":{"tokens":50,"fertility":2,"p90":2.68,"cpt":2.44,"neglect":0.11},"gemma":{"tokens":45,"fertility":1.82,"p90":2.42,"cpt":2.7,"neglect":0},"tekken":{"tokens":48,"fertility":1.92,"p90":2.58,"cpt":2.5,"neglect":0.07},"qwen2":{"tokens":49,"fertility":1.96,"p90":2.59,"cpt":2.47,"neglect":0.09},"bloom":{"tokens":45,"fertility":1.79,"p90":2.4,"cpt":2.71,"neglect":0}}}},{"code":"dzo_Tibt","name":"Dzongkha","script":"Tibetan","family":"Sino-Tibetan","region":"Eurasia","speakers":237080,"floor":100,"floors":{"flores":100},"metrics":{"flores":{"gpt2":{"tokens":396,"fertility":16.29,"p90":21.47,"cpt":0.37,"neglect":2.96},"cl100k":{"tokens":299,"fertility":12.27,"p90":16.24,"cpt":0.5,"neglect":1.99},"o200k":{"tokens":220,"fertility":9.11,"p90":11.95,"cpt":0.67,"neglect":1.2},"llama3":{"tokens":299,"fertility":12.27,"p90":16.27,"cpt":0.5,"neglect":1.99},"gemma":{"tokens":100,"fertility":4.13,"p90":5.43,"cpt":1.49,"neglect":0},"tekken":{"tokens":336,"fertility":13.64,"p90":17.83,"cpt":0.44,"neglect":2.36},"qwen2":{"tokens":201,"fertility":8.16,"p90":10.76,"cpt":0.74,"neglect":1.01},"bloom":{"tokens":181,"fertility":7.3,"p90":9.68,"cpt":0.82,"neglect":0.81}}}},{"code":"ell_Grek","name":"Modern Greek","script":"Greek","family":"Indo-European","region":"Eurasia","speakers":15000000,"floor":51,"floors":{"flores":51,"massive":14},"metrics":{"flores":{"gpt2":{"tokens":162,"fertility":6.53,"p90":8.34,"cpt":0.9,"neglect":2.18},"cl100k":{"tokens":128,"fertility":5.12,"p90":6.58,"cpt":1.14,"neglect":1.51},"o200k":{"tokens":51,"fertility":2.11,"p90":2.61,"cpt":2.76,"neglect":0},"llama3":{"tokens":56,"fertility":2.22,"p90":2.79,"cpt":2.6,"neglect":0.1},"gemma":{"tokens":56,"fertility":2.29,"p90":2.86,"cpt":2.55,"neglect":0.1},"tekken":{"tokens":53,"fertility":2.1,"p90":2.61,"cpt":2.7,"neglect":0.04},"qwen2":{"tokens":124,"fertility":4.94,"p90":6.37,"cpt":1.17,"neglect":1.43},"bloom":{"tokens":95,"fertility":3.8,"p90":4.71,"cpt":1.52,"neglect":0.86}},"massive":{"gpt2":{"tokens":42,"fertility":5.92,"p90":8.57,"cpt":0.92,"neglect":2},"cl100k":{"tokens":32,"fertility":4.71,"p90":7,"cpt":1.18,"neglect":1.29},"o200k":{"tokens":14,"fertility":2,"p90":3,"cpt":2.74,"neglect":0},"llama3":{"tokens":15,"fertility":2.14,"p90":3.1,"cpt":2.6,"neglect":0.07},"gemma":{"tokens":15,"fertility":2.2,"p90":3.29,"cpt":2.57,"neglect":0.07},"tekken":{"tokens":15,"fertility":2.06,"p90":3,"cpt":2.6,"neglect":0.07},"qwen2":{"tokens":32,"fertility":4.6,"p90":6.67,"cpt":1.21,"neglect":1.29},"bloom":{"tokens":24,"fertility":3.45,"p90":5,"cpt":1.57,"neglect":0.71}}}},{"code":"eng_Latn","name":"English","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":390000000,"floor":24,"floors":{"flores":24,"massive":7},"metrics":{"flores":{"gpt2":{"tokens":25,"fertility":1,"p90":1,"cpt":4.86,"neglect":0.04},"cl100k":{"tokens":25,"fertility":1,"p90":1,"cpt":4.85,"neglect":0.04},"o200k":{"tokens":24,"fertility":1,"p90":1,"cpt":4.9,"neglect":0},"llama3":{"tokens":25,"fertility":1,"p90":1,"cpt":4.85,"neglect":0.04},"gemma":{"tokens":25,"fertility":1,"p90":1,"cpt":4.85,"neglect":0.04},"tekken":{"tokens":25,"fertility":1,"p90":1,"cpt":4.73,"neglect":0.04},"qwen2":{"tokens":25,"fertility":1,"p90":1,"cpt":4.77,"neglect":0.04},"bloom":{"tokens":25,"fertility":1,"p90":1,"cpt":4.82,"neglect":0.04}},"massive":{"gpt2":{"tokens":7,"fertility":1,"p90":1,"cpt":4.58,"neglect":0},"cl100k":{"tokens":7,"fertility":1,"p90":1,"cpt":4.69,"neglect":0},"o200k":{"tokens":7,"fertility":1,"p90":1,"cpt":4.74,"neglect":0},"llama3":{"tokens":7,"fertility":1,"p90":1,"cpt":4.69,"neglect":0},"gemma":{"tokens":7,"fertility":1,"p90":1,"cpt":4.73,"neglect":0},"tekken":{"tokens":7,"fertility":1,"p90":1,"cpt":4.56,"neglect":0},"qwen2":{"tokens":7,"fertility":1,"p90":1,"cpt":4.69,"neglect":0},"bloom":{"tokens":7,"fertility":1,"p90":1,"cpt":4.59,"neglect":0}}}},{"code":"epo_Latn","name":"Esperanto","script":"Latin","family":"Artificial Language","region":"Eurasia","speakers":2000000,"floor":38,"floors":{"flores":38},"metrics":{"flores":{"gpt2":{"tokens":50,"fertility":2,"p90":2.49,"cpt":2.4,"neglect":0.32},"cl100k":{"tokens":46,"fertility":1.87,"p90":2.32,"cpt":2.59,"neglect":0.21},"o200k":{"tokens":38,"fertility":1.53,"p90":1.84,"cpt":3.19,"neglect":0},"llama3":{"tokens":46,"fertility":1.85,"p90":2.29,"cpt":2.62,"neglect":0.21},"gemma":{"tokens":39,"fertility":1.58,"p90":1.92,"cpt":3.08,"neglect":0.03},"tekken":{"tokens":39,"fertility":1.52,"p90":1.81,"cpt":3.12,"neglect":0.03},"qwen2":{"tokens":45,"fertility":1.79,"p90":2.21,"cpt":2.66,"neglect":0.18},"bloom":{"tokens":41,"fertility":1.64,"p90":2,"cpt":2.92,"neglect":0.08}}}},{"code":"est_Latn","name":"Estonian","script":"Latin","family":"Uralic","region":"Eurasia","speakers":1100000,"floor":36,"floors":{"flores":36},"metrics":{"flores":{"gpt2":{"tokens":52,"fertility":2.09,"p90":2.64,"cpt":2.27,"neglect":0.44},"cl100k":{"tokens":46,"fertility":1.86,"p90":2.31,"cpt":2.55,"neglect":0.28},"o200k":{"tokens":36,"fertility":1.48,"p90":1.82,"cpt":3.24,"neglect":0},"llama3":{"tokens":46,"fertility":1.84,"p90":2.3,"cpt":2.57,"neglect":0.28},"gemma":{"tokens":40,"fertility":1.61,"p90":2,"cpt":2.96,"neglect":0.11},"tekken":{"tokens":41,"fertility":1.59,"p90":1.97,"cpt":2.92,"neglect":0.14},"qwen2":{"tokens":46,"fertility":1.84,"p90":2.3,"cpt":2.54,"neglect":0.28},"bloom":{"tokens":44,"fertility":1.77,"p90":2.17,"cpt":2.68,"neglect":0.22}}}},{"code":"eus_Latn","name":"Basque","script":"Latin","family":"Isolate","region":"Eurasia","speakers":750000,"floor":29,"floors":{"flores":29},"metrics":{"flores":{"gpt2":{"tokens":51,"fertility":2.08,"p90":2.63,"cpt":2.47,"neglect":0.76},"cl100k":{"tokens":47,"fertility":1.88,"p90":2.38,"cpt":2.74,"neglect":0.62},"o200k":{"tokens":39,"fertility":1.6,"p90":2,"cpt":3.25,"neglect":0.34},"llama3":{"tokens":46,"fertility":1.87,"p90":2.35,"cpt":2.76,"neglect":0.59},"gemma":{"tokens":42,"fertility":1.68,"p90":2.1,"cpt":3.06,"neglect":0.45},"tekken":{"tokens":39,"fertility":1.53,"p90":1.9,"cpt":3.28,"neglect":0.34},"qwen2":{"tokens":47,"fertility":1.87,"p90":2.36,"cpt":2.71,"neglect":0.62},"bloom":{"tokens":29,"fertility":1.14,"p90":1.41,"cpt":4.46,"neglect":0}}}},{"code":"ewe_Latn","name":"Ewe","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":3000000,"floor":52,"floors":{"flores":52},"metrics":{"flores":{"gpt2":{"tokens":70,"fertility":2.86,"p90":4.19,"cpt":1.64,"neglect":0.35},"cl100k":{"tokens":67,"fertility":2.72,"p90":4,"cpt":1.72,"neglect":0.29},"o200k":{"tokens":57,"fertility":2.33,"p90":3.38,"cpt":2.03,"neglect":0.1},"llama3":{"tokens":64,"fertility":2.62,"p90":3.83,"cpt":1.79,"neglect":0.23},"gemma":{"tokens":54,"fertility":2.17,"p90":3.18,"cpt":2.14,"neglect":0.04},"tekken":{"tokens":65,"fertility":2.6,"p90":3.82,"cpt":1.75,"neglect":0.25},"qwen2":{"tokens":58,"fertility":2.32,"p90":3.36,"cpt":1.98,"neglect":0.12},"bloom":{"tokens":52,"fertility":2.1,"p90":3,"cpt":2.23,"neglect":0}}}},{"code":"fao_Latn","name":"Faroese","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":69150,"floor":40,"floors":{"flores":40},"metrics":{"flores":{"gpt2":{"tokens":59,"fertility":2.39,"p90":2.93,"cpt":2.09,"neglect":0.47},"cl100k":{"tokens":51,"fertility":2.06,"p90":2.57,"cpt":2.4,"neglect":0.28},"o200k":{"tokens":40,"fertility":1.64,"p90":2,"cpt":3.05,"neglect":0},"llama3":{"tokens":50,"fertility":2.04,"p90":2.52,"cpt":2.44,"neglect":0.25},"gemma":{"tokens":45,"fertility":1.83,"p90":2.28,"cpt":2.71,"neglect":0.12},"tekken":{"tokens":47,"fertility":1.86,"p90":2.31,"cpt":2.6,"neglect":0.17},"qwen2":{"tokens":51,"fertility":2.04,"p90":2.54,"cpt":2.4,"neglect":0.28},"bloom":{"tokens":48,"fertility":1.95,"p90":2.39,"cpt":2.52,"neglect":0.2}}}},{"code":"fij_Latn","name":"Fijian","script":"Latin","family":"Austronesian","region":"Papunesia","speakers":341270,"floor":44,"floors":{"flores":44},"metrics":{"flores":{"gpt2":{"tokens":56,"fertility":2.26,"p90":2.95,"cpt":2.49,"neglect":0.27},"cl100k":{"tokens":52,"fertility":2.11,"p90":2.75,"cpt":2.66,"neglect":0.18},"o200k":{"tokens":44,"fertility":1.82,"p90":2.39,"cpt":3.13,"neglect":0},"llama3":{"tokens":52,"fertility":2.09,"p90":2.74,"cpt":2.68,"neglect":0.18},"gemma":{"tokens":47,"fertility":1.88,"p90":2.5,"cpt":2.98,"neglect":0.07},"tekken":{"tokens":51,"fertility":2,"p90":2.66,"cpt":2.72,"neglect":0.16},"qwen2":{"tokens":52,"fertility":2.09,"p90":2.75,"cpt":2.64,"neglect":0.18},"bloom":{"tokens":48,"fertility":1.95,"p90":2.57,"cpt":2.86,"neglect":0.09}}}},{"code":"fin_Latn","name":"Finnish","script":"Latin","family":"Uralic","region":"Eurasia","speakers":5413380,"floor":38,"floors":{"flores":38,"massive":10},"metrics":{"flores":{"gpt2":{"tokens":56,"fertility":2.28,"p90":2.86,"cpt":2.29,"neglect":0.47},"cl100k":{"tokens":50,"fertility":2,"p90":2.5,"cpt":2.6,"neglect":0.32},"o200k":{"tokens":38,"fertility":1.55,"p90":1.92,"cpt":3.37,"neglect":0},"llama3":{"tokens":49,"fertility":2,"p90":2.5,"cpt":2.62,"neglect":0.29},"gemma":{"tokens":40,"fertility":1.61,"p90":2,"cpt":3.22,"neglect":0.05},"tekken":{"tokens":42,"fertility":1.64,"p90":2.05,"cpt":3.07,"neglect":0.11},"qwen2":{"tokens":50,"fertility":1.97,"p90":2.5,"cpt":2.59,"neglect":0.32},"bloom":{"tokens":47,"fertility":1.89,"p90":2.35,"cpt":2.72,"neglect":0.24}},"massive":{"gpt2":{"tokens":15,"fertility":2.11,"p90":3.17,"cpt":2.29,"neglect":0.5},"cl100k":{"tokens":13,"fertility":1.87,"p90":2.75,"cpt":2.65,"neglect":0.3},"o200k":{"tokens":10,"fertility":1.5,"p90":2,"cpt":3.44,"neglect":0},"llama3":{"tokens":13,"fertility":1.86,"p90":2.75,"cpt":2.67,"neglect":0.3},"gemma":{"tokens":10,"fertility":1.5,"p90":2.17,"cpt":3.34,"neglect":0},"tekken":{"tokens":11,"fertility":1.57,"p90":2.25,"cpt":3.09,"neglect":0.1},"qwen2":{"tokens":13,"fertility":1.86,"p90":2.75,"cpt":2.66,"neglect":0.3},"bloom":{"tokens":12,"fertility":1.75,"p90":2.6,"cpt":2.75,"neglect":0.2}}}},{"code":"fon_Latn","name":"Fon","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":1935500,"floor":54,"floors":{"flores":54},"metrics":{"flores":{"gpt2":{"tokens":98,"fertility":3.93,"p90":5.7,"cpt":1.23,"neglect":0.81},"cl100k":{"tokens":90,"fertility":3.56,"p90":5.08,"cpt":1.36,"neglect":0.67},"o200k":{"tokens":70,"fertility":2.79,"p90":3.95,"cpt":1.75,"neglect":0.3},"llama3":{"tokens":89,"fertility":3.54,"p90":5.04,"cpt":1.38,"neglect":0.65},"gemma":{"tokens":70,"fertility":2.8,"p90":3.92,"cpt":1.75,"neglect":0.3},"tekken":{"tokens":88,"fertility":3.46,"p90":5,"cpt":1.37,"neglect":0.63},"qwen2":{"tokens":72,"fertility":2.8,"p90":3.93,"cpt":1.71,"neglect":0.33},"bloom":{"tokens":54,"fertility":2.12,"p90":3.05,"cpt":2.24,"neglect":0}}}},{"code":"fra_Latn","name":"French","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":208157220,"floor":30,"floors":{"flores":30,"massive":8},"metrics":{"flores":{"gpt2":{"tokens":49,"fertility":1.97,"p90":2.47,"cpt":2.92,"neglect":0.63},"cl100k":{"tokens":39,"fertility":1.57,"p90":1.94,"cpt":3.64,"neglect":0.3},"o200k":{"tokens":33,"fertility":1.33,"p90":1.62,"cpt":4.33,"neglect":0.1},"llama3":{"tokens":39,"fertility":1.56,"p90":1.94,"cpt":3.66,"neglect":0.3},"gemma":{"tokens":34,"fertility":1.33,"p90":1.63,"cpt":4.29,"neglect":0.13},"tekken":{"tokens":33,"fertility":1.29,"p90":1.56,"cpt":4.31,"neglect":0.1},"qwen2":{"tokens":39,"fertility":1.54,"p90":1.92,"cpt":3.64,"neglect":0.3},"bloom":{"tokens":30,"fertility":1.18,"p90":1.43,"cpt":4.8,"neglect":0}},"massive":{"gpt2":{"tokens":14,"fertility":2,"p90":3,"cpt":2.84,"neglect":0.75},"cl100k":{"tokens":11,"fertility":1.55,"p90":2.25,"cpt":3.7,"neglect":0.38},"o200k":{"tokens":9,"fertility":1.33,"p90":2,"cpt":4.38,"neglect":0.12},"llama3":{"tokens":11,"fertility":1.54,"p90":2.25,"cpt":3.71,"neglect":0.38},"gemma":{"tokens":9,"fertility":1.33,"p90":2,"cpt":4.28,"neglect":0.12},"tekken":{"tokens":9,"fertility":1.33,"p90":2,"cpt":4.24,"neglect":0.12},"qwen2":{"tokens":10,"fertility":1.5,"p90":2.25,"cpt":3.74,"neglect":0.25},"bloom":{"tokens":8,"fertility":1.2,"p90":1.67,"cpt":4.73,"neglect":0}}}},{"code":"fur_Latn","name":"Friulian","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":300000,"floor":42,"floors":{"flores":42},"metrics":{"flores":{"gpt2":{"tokens":51,"fertility":2.05,"p90":2.54,"cpt":2.6,"neglect":0.21},"cl100k":{"tokens":46,"fertility":1.83,"p90":2.26,"cpt":2.89,"neglect":0.1},"o200k":{"tokens":42,"fertility":1.71,"p90":2.11,"cpt":3.12,"neglect":0},"llama3":{"tokens":45,"fertility":1.81,"p90":2.24,"cpt":2.92,"neglect":0.07},"gemma":{"tokens":42,"fertility":1.68,"p90":2.07,"cpt":3.16,"neglect":0},"tekken":{"tokens":44,"fertility":1.7,"p90":2.11,"cpt":3.04,"neglect":0.05},"qwen2":{"tokens":46,"fertility":1.82,"p90":2.26,"cpt":2.87,"neglect":0.1},"bloom":{"tokens":43,"fertility":1.69,"p90":2.06,"cpt":3.11,"neglect":0.02}}}},{"code":"fuv_Latn","name":"Hausa States Fulfulde","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":14500000,"floor":40,"floors":{"flores":40},"metrics":{"flores":{"gpt2":{"tokens":49,"fertility":1.94,"p90":2.8,"cpt":2.3,"neglect":0.23},"cl100k":{"tokens":46,"fertility":1.8,"p90":2.59,"cpt":2.47,"neglect":0.15},"o200k":{"tokens":40,"fertility":1.59,"p90":2.17,"cpt":2.86,"neglect":0},"llama3":{"tokens":45,"fertility":1.78,"p90":2.57,"cpt":2.5,"neglect":0.12},"gemma":{"tokens":41,"fertility":1.6,"p90":2.24,"cpt":2.78,"neglect":0.03},"tekken":{"tokens":44,"fertility":1.7,"p90":2.42,"cpt":2.56,"neglect":0.1},"qwen2":{"tokens":45,"fertility":1.73,"p90":2.38,"cpt":2.56,"neglect":0.12},"bloom":{"tokens":42,"fertility":1.62,"p90":2.26,"cpt":2.73,"neglect":0.05}}}},{"code":"gaz_Latn","name":"West Central Oromo","script":"Latin","family":"Afro-Asiatic","region":"Africa","speakers":24000000,"floor":46,"floors":{"flores":46},"metrics":{"flores":{"gpt2":{"tokens":63,"fertility":2.51,"p90":3.2,"cpt":2.29,"neglect":0.37},"cl100k":{"tokens":58,"fertility":2.3,"p90":2.92,"cpt":2.49,"neglect":0.26},"o200k":{"tokens":46,"fertility":1.88,"p90":2.33,"cpt":3.09,"neglect":0},"llama3":{"tokens":57,"fertility":2.29,"p90":2.91,"cpt":2.5,"neglect":0.24},"gemma":{"tokens":52,"fertility":2.08,"p90":2.69,"cpt":2.75,"neglect":0.13},"tekken":{"tokens":57,"fertility":2.21,"p90":2.83,"cpt":2.53,"neglect":0.24},"qwen2":{"tokens":58,"fertility":2.29,"p90":2.91,"cpt":2.47,"neglect":0.26},"bloom":{"tokens":55,"fertility":2.18,"p90":2.74,"cpt":2.61,"neglect":0.2}}}},{"code":"gla_Latn","name":"Scottish Gaelic","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":60130,"floor":49,"floors":{"flores":49},"metrics":{"flores":{"gpt2":{"tokens":67,"fertility":2.67,"p90":3.48,"cpt":2.25,"neglect":0.37},"cl100k":{"tokens":60,"fertility":2.41,"p90":3.1,"cpt":2.49,"neglect":0.22},"o200k":{"tokens":49,"fertility":1.97,"p90":2.5,"cpt":3.06,"neglect":0},"llama3":{"tokens":60,"fertility":2.39,"p90":3.08,"cpt":2.51,"neglect":0.22},"gemma":{"tokens":56,"fertility":2.26,"p90":2.93,"cpt":2.67,"neglect":0.14},"tekken":{"tokens":59,"fertility":2.29,"p90":3,"cpt":2.54,"neglect":0.2},"qwen2":{"tokens":60,"fertility":2.38,"p90":3.08,"cpt":2.49,"neglect":0.22},"bloom":{"tokens":56,"fertility":2.24,"p90":2.88,"cpt":2.66,"neglect":0.14}}}},{"code":"gle_Latn","name":"Irish","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":141000,"floor":45,"floors":{"flores":45},"metrics":{"flores":{"gpt2":{"tokens":63,"fertility":2.55,"p90":3.28,"cpt":2.22,"neglect":0.4},"cl100k":{"tokens":58,"fertility":2.32,"p90":3,"cpt":2.42,"neglect":0.29},"o200k":{"tokens":45,"fertility":1.83,"p90":2.3,"cpt":3.07,"neglect":0},"llama3":{"tokens":57,"fertility":2.27,"p90":2.92,"cpt":2.48,"neglect":0.27},"gemma":{"tokens":53,"fertility":2.11,"p90":2.72,"cpt":2.65,"neglect":0.18},"tekken":{"tokens":56,"fertility":2.17,"p90":2.8,"cpt":2.53,"neglect":0.24},"qwen2":{"tokens":58,"fertility":2.29,"p90":2.95,"cpt":2.43,"neglect":0.29},"bloom":{"tokens":54,"fertility":2.15,"p90":2.72,"cpt":2.59,"neglect":0.2}}}},{"code":"glg_Latn","name":"Galician","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":2400000,"floor":32,"floors":{"flores":32},"metrics":{"flores":{"gpt2":{"tokens":48,"fertility":1.91,"p90":2.35,"cpt":2.83,"neglect":0.5},"cl100k":{"tokens":39,"fertility":1.56,"p90":1.88,"cpt":3.45,"neglect":0.22},"o200k":{"tokens":32,"fertility":1.31,"p90":1.59,"cpt":4.09,"neglect":0},"llama3":{"tokens":39,"fertility":1.55,"p90":1.88,"cpt":3.46,"neglect":0.22},"gemma":{"tokens":33,"fertility":1.3,"p90":1.58,"cpt":4.09,"neglect":0.03},"tekken":{"tokens":33,"fertility":1.3,"p90":1.58,"cpt":3.98,"neglect":0.03},"qwen2":{"tokens":39,"fertility":1.54,"p90":1.86,"cpt":3.44,"neglect":0.22},"bloom":{"tokens":32,"fertility":1.25,"p90":1.5,"cpt":4.21,"neglect":0}}}},{"code":"grn_Latn","name":"Guarani","script":"Latin","family":"Tupian","region":"South America","speakers":6500000,"floor":43,"floors":{"flores":43},"metrics":{"flores":{"gpt2":{"tokens":61,"fertility":2.43,"p90":3.22,"cpt":2.01,"neglect":0.42},"cl100k":{"tokens":54,"fertility":2.16,"p90":2.82,"cpt":2.26,"neglect":0.26},"o200k":{"tokens":43,"fertility":1.76,"p90":2.26,"cpt":2.79,"neglect":0},"llama3":{"tokens":52,"fertility":2.07,"p90":2.68,"cpt":2.36,"neglect":0.21},"gemma":{"tokens":46,"fertility":1.87,"p90":2.44,"cpt":2.6,"neglect":0.07},"tekken":{"tokens":49,"fertility":1.93,"p90":2.53,"cpt":2.46,"neglect":0.14},"qwen2":{"tokens":53,"fertility":2.06,"p90":2.71,"cpt":2.31,"neglect":0.23},"bloom":{"tokens":47,"fertility":1.85,"p90":2.41,"cpt":2.6,"neglect":0.09}}}},{"code":"guj_Gujr","name":"Gujarati","script":"Gujarati","family":"Indo-European","region":"Eurasia","speakers":56400000,"floor":34,"floors":{"flores":34},"metrics":{"flores":{"gpt2":{"tokens":303,"fertility":12.04,"p90":15.29,"cpt":0.38,"neglect":7.91},"cl100k":{"tokens":191,"fertility":7.62,"p90":9.66,"cpt":0.61,"neglect":4.62},"o200k":{"tokens":44,"fertility":1.78,"p90":2.17,"cpt":2.63,"neglect":0.29},"llama3":{"tokens":191,"fertility":7.62,"p90":9.66,"cpt":0.61,"neglect":4.62},"gemma":{"tokens":75,"fertility":3,"p90":3.75,"cpt":1.55,"neglect":1.21},"tekken":{"tokens":70,"fertility":2.74,"p90":3.4,"cpt":1.66,"neglect":1.06},"qwen2":{"tokens":171,"fertility":6.74,"p90":8.56,"cpt":0.68,"neglect":4.03},"bloom":{"tokens":34,"fertility":1.36,"p90":1.65,"cpt":3.4,"neglect":0}}}},{"code":"hat_Latn","name":"Haitian","script":"Latin","family":"Indo-European","region":"North America","speakers":9600000,"floor":35,"floors":{"flores":35},"metrics":{"flores":{"gpt2":{"tokens":47,"fertility":1.89,"p90":2.42,"cpt":2.35,"neglect":0.34},"cl100k":{"tokens":43,"fertility":1.73,"p90":2.18,"cpt":2.57,"neglect":0.23},"o200k":{"tokens":35,"fertility":1.4,"p90":1.73,"cpt":3.21,"neglect":0},"llama3":{"tokens":42,"fertility":1.69,"p90":2.14,"cpt":2.64,"neglect":0.2},"gemma":{"tokens":38,"fertility":1.53,"p90":1.89,"cpt":2.93,"neglect":0.09},"tekken":{"tokens":41,"fertility":1.59,"p90":2,"cpt":2.73,"neglect":0.17},"qwen2":{"tokens":44,"fertility":1.71,"p90":2.17,"cpt":2.55,"neglect":0.26},"bloom":{"tokens":39,"fertility":1.55,"p90":1.94,"cpt":2.84,"neglect":0.11}}}},{"code":"hau_Latn","name":"Hausa","script":"Latin","family":"Afro-Asiatic","region":"Africa","speakers":43900000,"floor":39,"floors":{"flores":39},"metrics":{"flores":{"gpt2":{"tokens":54,"fertility":2.16,"p90":2.74,"cpt":2.42,"neglect":0.38},"cl100k":{"tokens":50,"fertility":2,"p90":2.54,"cpt":2.59,"neglect":0.28},"o200k":{"tokens":39,"fertility":1.57,"p90":1.93,"cpt":3.38,"neglect":0},"llama3":{"tokens":50,"fertility":2,"p90":2.52,"cpt":2.62,"neglect":0.28},"gemma":{"tokens":43,"fertility":1.74,"p90":2.17,"cpt":3.01,"neglect":0.1},"tekken":{"tokens":49,"fertility":1.91,"p90":2.43,"cpt":2.66,"neglect":0.26},"qwen2":{"tokens":50,"fertility":1.96,"p90":2.45,"cpt":2.61,"neglect":0.28},"bloom":{"tokens":45,"fertility":1.78,"p90":2.23,"cpt":2.89,"neglect":0.15}}}},{"code":"heb_Hebr","name":"Modern Hebrew","script":"Hebrew","family":"Afro-Asiatic","region":"Eurasia","speakers":9303950,"floor":36,"floors":{"flores":36,"massive":10},"metrics":{"flores":{"gpt2":{"tokens":109,"fertility":4.33,"p90":5.33,"cpt":0.87,"neglect":2.03},"cl100k":{"tokens":91,"fertility":3.64,"p90":4.48,"cpt":1.03,"neglect":1.53},"o200k":{"tokens":36,"fertility":1.47,"p90":1.79,"cpt":2.57,"neglect":0},"llama3":{"tokens":91,"fertility":3.64,"p90":4.48,"cpt":1.03,"neglect":1.53},"gemma":{"tokens":40,"fertility":1.61,"p90":1.97,"cpt":2.32,"neglect":0.11},"tekken":{"tokens":41,"fertility":1.62,"p90":1.97,"cpt":2.26,"neglect":0.14},"qwen2":{"tokens":37,"fertility":1.46,"p90":1.8,"cpt":2.52,"neglect":0.03},"bloom":{"tokens":73,"fertility":2.9,"p90":3.5,"cpt":1.29,"neglect":1.03}},"massive":{"gpt2":{"tokens":30,"fertility":4.38,"p90":6.2,"cpt":0.86,"neglect":2},"cl100k":{"tokens":25,"fertility":3.71,"p90":5.29,"cpt":1.04,"neglect":1.5},"o200k":{"tokens":10,"fertility":1.5,"p90":2.2,"cpt":2.64,"neglect":0},"llama3":{"tokens":25,"fertility":3.71,"p90":5.29,"cpt":1.04,"neglect":1.5},"gemma":{"tokens":11,"fertility":1.64,"p90":2.4,"cpt":2.41,"neglect":0.1},"tekken":{"tokens":12,"fertility":1.7,"p90":2.5,"cpt":2.19,"neglect":0.2},"qwen2":{"tokens":10,"fertility":1.43,"p90":2,"cpt":2.7,"neglect":0},"bloom":{"tokens":21,"fertility":3,"p90":4.25,"cpt":1.24,"neglect":1.1}}}},{"code":"hin_Deva","name":"Hindi","script":"Devanagari","family":"Indo-European","region":"Eurasia","speakers":341000000,"floor":33,"floors":{"flores":33,"massive":9},"metrics":{"flores":{"gpt2":{"tokens":185,"fertility":7.39,"p90":9.42,"cpt":0.65,"neglect":4.61},"cl100k":{"tokens":120,"fertility":4.81,"p90":6,"cpt":1.01,"neglect":2.64},"o200k":{"tokens":38,"fertility":1.57,"p90":1.94,"cpt":3.11,"neglect":0.15},"llama3":{"tokens":63,"fertility":2.52,"p90":3.16,"cpt":1.92,"neglect":0.91},"gemma":{"tokens":46,"fertility":1.85,"p90":2.31,"cpt":2.61,"neglect":0.39},"tekken":{"tokens":46,"fertility":1.79,"p90":2.21,"cpt":2.64,"neglect":0.39},"qwen2":{"tokens":113,"fertility":4.5,"p90":5.62,"cpt":1.07,"neglect":2.42},"bloom":{"tokens":33,"fertility":1.29,"p90":1.56,"cpt":3.72,"neglect":0}},"massive":{"gpt2":{"tokens":50,"fertility":7.25,"p90":10.25,"cpt":0.66,"neglect":4.56},"cl100k":{"tokens":32,"fertility":4.75,"p90":6.6,"cpt":1.02,"neglect":2.56},"o200k":{"tokens":10,"fertility":1.54,"p90":2.3,"cpt":3.16,"neglect":0.11},"llama3":{"tokens":18,"fertility":2.62,"p90":3.67,"cpt":1.86,"neglect":1},"gemma":{"tokens":12,"fertility":1.83,"p90":2.71,"cpt":2.67,"neglect":0.33},"tekken":{"tokens":13,"fertility":1.8,"p90":2.67,"cpt":2.61,"neglect":0.44},"qwen2":{"tokens":30,"fertility":4.43,"p90":6.17,"cpt":1.1,"neglect":2.33},"bloom":{"tokens":9,"fertility":1.25,"p90":1.78,"cpt":3.78,"neglect":0}}}},{"code":"hne_Deva","name":"Chhattisgarhi","script":"Devanagari","family":"Indo-European","region":"Eurasia","speakers":16300000,"floor":37,"floors":{"flores":37},"metrics":{"flores":{"gpt2":{"tokens":178,"fertility":7.18,"p90":8.91,"cpt":0.66,"neglect":3.81},"cl100k":{"tokens":117,"fertility":4.7,"p90":5.78,"cpt":1,"neglect":2.16},"o200k":{"tokens":43,"fertility":1.74,"p90":2.13,"cpt":2.71,"neglect":0.16},"llama3":{"tokens":62,"fertility":2.5,"p90":3,"cpt":1.89,"neglect":0.68},"gemma":{"tokens":52,"fertility":2.08,"p90":2.58,"cpt":2.25,"neglect":0.41},"tekken":{"tokens":52,"fertility":2.03,"p90":2.48,"cpt":2.25,"neglect":0.41},"qwen2":{"tokens":107,"fertility":4.26,"p90":5.28,"cpt":1.09,"neglect":1.89},"bloom":{"tokens":37,"fertility":1.46,"p90":1.79,"cpt":3.19,"neglect":0}}}},{"code":"hrv_Latn","name":"Croatian","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":5600000,"floor":36,"floors":{"flores":36},"metrics":{"flores":{"gpt2":{"tokens":54,"fertility":2.12,"p90":2.73,"cpt":2.24,"neglect":0.5},"cl100k":{"tokens":46,"fertility":1.83,"p90":2.33,"cpt":2.59,"neglect":0.28},"o200k":{"tokens":36,"fertility":1.47,"p90":1.85,"cpt":3.27,"neglect":0},"llama3":{"tokens":44,"fertility":1.76,"p90":2.24,"cpt":2.7,"neglect":0.22},"gemma":{"tokens":39,"fertility":1.59,"p90":2,"cpt":3,"neglect":0.08},"tekken":{"tokens":38,"fertility":1.48,"p90":1.88,"cpt":3.12,"neglect":0.06},"qwen2":{"tokens":46,"fertility":1.82,"p90":2.33,"cpt":2.57,"neglect":0.28},"bloom":{"tokens":46,"fertility":1.8,"p90":2.28,"cpt":2.63,"neglect":0.28}}}},{"code":"hun_Latn","name":"Hungarian","script":"Latin","family":"Uralic","region":"Eurasia","speakers":12600000,"floor":40,"floors":{"flores":40,"massive":11},"metrics":{"flores":{"gpt2":{"tokens":65,"fertility":2.6,"p90":3.42,"cpt":1.95,"neglect":0.62},"cl100k":{"tokens":52,"fertility":2.11,"p90":2.74,"cpt":2.39,"neglect":0.3},"o200k":{"tokens":43,"fertility":1.78,"p90":2.25,"cpt":2.9,"neglect":0.07},"llama3":{"tokens":51,"fertility":2.08,"p90":2.68,"cpt":2.44,"neglect":0.28},"gemma":{"tokens":41,"fertility":1.64,"p90":2.11,"cpt":3.09,"neglect":0.03},"tekken":{"tokens":40,"fertility":1.59,"p90":2.03,"cpt":3.1,"neglect":0},"qwen2":{"tokens":52,"fertility":2.09,"p90":2.71,"cpt":2.38,"neglect":0.3},"bloom":{"tokens":51,"fertility":2.04,"p90":2.58,"cpt":2.47,"neglect":0.28}},"massive":{"gpt2":{"tokens":17,"fertility":2.5,"p90":3.89,"cpt":1.96,"neglect":0.55},"cl100k":{"tokens":14,"fertility":2.08,"p90":3.25,"cpt":2.39,"neglect":0.27},"o200k":{"tokens":12,"fertility":1.75,"p90":2.67,"cpt":2.88,"neglect":0.09},"llama3":{"tokens":14,"fertility":2,"p90":3.17,"cpt":2.44,"neglect":0.27},"gemma":{"tokens":11,"fertility":1.67,"p90":2.57,"cpt":3.07,"neglect":0},"tekken":{"tokens":12,"fertility":1.67,"p90":2.5,"cpt":2.92,"neglect":0.09},"qwen2":{"tokens":14,"fertility":2,"p90":3.2,"cpt":2.4,"neglect":0.27},"bloom":{"tokens":14,"fertility":2,"p90":3,"cpt":2.46,"neglect":0.27}}}},{"code":"hye_Armn","name":"Eastern Armenian","script":"Armenian","family":"Indo-European","region":"Eurasia","speakers":6700000,"floor":44,"floors":{"flores":44,"massive":12},"metrics":{"flores":{"gpt2":{"tokens":247,"fertility":9.88,"p90":12.87,"cpt":0.54,"neglect":4.61},"cl100k":{"tokens":247,"fertility":9.92,"p90":12.91,"cpt":0.54,"neglect":4.61},"o200k":{"tokens":44,"fertility":1.78,"p90":2.2,"cpt":3.04,"neglect":0},"llama3":{"tokens":217,"fertility":8.65,"p90":11.33,"cpt":0.62,"neglect":3.93},"gemma":{"tokens":76,"fertility":3.04,"p90":3.82,"cpt":1.77,"neglect":0.73},"tekken":{"tokens":49,"fertility":1.9,"p90":2.36,"cpt":2.75,"neglect":0.11},"qwen2":{"tokens":133,"fertility":5.31,"p90":6.92,"cpt":1,"neglect":2.02},"bloom":{"tokens":107,"fertility":4.27,"p90":5.38,"cpt":1.24,"neglect":1.43}},"massive":{"gpt2":{"tokens":63,"fertility":9,"p90":13.6,"cpt":0.54,"neglect":4.25},"cl100k":{"tokens":63,"fertility":9.17,"p90":14,"cpt":0.54,"neglect":4.25},"o200k":{"tokens":12,"fertility":1.78,"p90":2.67,"cpt":2.85,"neglect":0},"llama3":{"tokens":56,"fertility":8.17,"p90":12.5,"cpt":0.61,"neglect":3.67},"gemma":{"tokens":20,"fertility":3,"p90":4.4,"cpt":1.72,"neglect":0.67},"tekken":{"tokens":13,"fertility":1.9,"p90":2.8,"cpt":2.55,"neglect":0.08},"qwen2":{"tokens":34,"fertility":4.9,"p90":7.33,"cpt":1.01,"neglect":1.83},"bloom":{"tokens":28,"fertility":4,"p90":5.86,"cpt":1.22,"neglect":1.33}}}},{"code":"ibo_Latn","name":"Igbo","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":27000000,"floor":41,"floors":{"flores":41},"metrics":{"flores":{"gpt2":{"tokens":82,"fertility":3.32,"p90":4.68,"cpt":1.47,"neglect":1},"cl100k":{"tokens":59,"fertility":2.38,"p90":3.15,"cpt":2.04,"neglect":0.44},"o200k":{"tokens":41,"fertility":1.62,"p90":2.07,"cpt":3.02,"neglect":0},"llama3":{"tokens":58,"fertility":2.33,"p90":3.06,"cpt":2.09,"neglect":0.41},"gemma":{"tokens":53,"fertility":2.15,"p90":2.88,"cpt":2.26,"neglect":0.29},"tekken":{"tokens":59,"fertility":2.3,"p90":3.06,"cpt":2.06,"neglect":0.44},"qwen2":{"tokens":58,"fertility":2.31,"p90":3.05,"cpt":2.07,"neglect":0.41},"bloom":{"tokens":42,"fertility":1.67,"p90":2.12,"cpt":2.88,"neglect":0.02}}}},{"code":"ilo_Latn","name":"Iloko","script":"Latin","family":"Austronesian","region":"Papunesia","speakers":9100000,"floor":45,"floors":{"flores":45},"metrics":{"flores":{"gpt2":{"tokens":56,"fertility":2.25,"p90":2.83,"cpt":2.62,"neglect":0.24},"cl100k":{"tokens":51,"fertility":2.05,"p90":2.57,"cpt":2.87,"neglect":0.13},"o200k":{"tokens":45,"fertility":1.83,"p90":2.29,"cpt":3.23,"neglect":0},"llama3":{"tokens":51,"fertility":2.04,"p90":2.55,"cpt":2.89,"neglect":0.13},"gemma":{"tokens":45,"fertility":1.83,"p90":2.32,"cpt":3.2,"neglect":0},"tekken":{"tokens":50,"fertility":1.96,"p90":2.47,"cpt":2.93,"neglect":0.11},"qwen2":{"tokens":52,"fertility":2.04,"p90":2.57,"cpt":2.85,"neglect":0.16},"bloom":{"tokens":47,"fertility":1.89,"p90":2.35,"cpt":3.07,"neglect":0.04}}}},{"code":"ind_Latn","name":"Standard Indonesian","script":"Latin","family":"Austronesian","region":"Papunesia","speakers":198996550,"floor":24,"floors":{"flores":24,"massive":7},"metrics":{"flores":{"gpt2":{"tokens":49,"fertility":1.97,"p90":2.45,"cpt":2.68,"neglect":1.04},"cl100k":{"tokens":38,"fertility":1.55,"p90":1.9,"cpt":3.41,"neglect":0.58},"o200k":{"tokens":30,"fertility":1.23,"p90":1.5,"cpt":4.32,"neglect":0.25},"llama3":{"tokens":38,"fertility":1.55,"p90":1.88,"cpt":3.43,"neglect":0.58},"gemma":{"tokens":27,"fertility":1.1,"p90":1.33,"cpt":4.79,"neglect":0.12},"tekken":{"tokens":33,"fertility":1.31,"p90":1.61,"cpt":3.93,"neglect":0.38},"qwen2":{"tokens":39,"fertility":1.55,"p90":1.88,"cpt":3.37,"neglect":0.62},"bloom":{"tokens":24,"fertility":0.97,"p90":1.15,"cpt":5.39,"neglect":0}},"massive":{"gpt2":{"tokens":13,"fertility":1.86,"p90":2.67,"cpt":2.71,"neglect":0.86},"cl100k":{"tokens":10,"fertility":1.44,"p90":2,"cpt":3.52,"neglect":0.43},"o200k":{"tokens":8,"fertility":1.2,"p90":1.67,"cpt":4.32,"neglect":0.14},"llama3":{"tokens":10,"fertility":1.43,"p90":2,"cpt":3.54,"neglect":0.43},"gemma":{"tokens":7,"fertility":1,"p90":1.5,"cpt":4.84,"neglect":0},"tekken":{"tokens":9,"fertility":1.33,"p90":1.9,"cpt":3.79,"neglect":0.29},"qwen2":{"tokens":10,"fertility":1.44,"p90":2,"cpt":3.52,"neglect":0.43},"bloom":{"tokens":7,"fertility":1,"p90":1.43,"cpt":5.02,"neglect":0}}}},{"code":"isl_Latn","name":"Icelandic","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":321040,"floor":40,"floors":{"flores":40,"massive":11},"metrics":{"flores":{"gpt2":{"tokens":60,"fertility":2.42,"p90":3.04,"cpt":1.99,"neglect":0.5},"cl100k":{"tokens":54,"fertility":2.16,"p90":2.69,"cpt":2.24,"neglect":0.35},"o200k":{"tokens":40,"fertility":1.6,"p90":2,"cpt":3.02,"neglect":0},"llama3":{"tokens":52,"fertility":2.11,"p90":2.64,"cpt":2.29,"neglect":0.3},"gemma":{"tokens":46,"fertility":1.86,"p90":2.33,"cpt":2.59,"neglect":0.15},"tekken":{"tokens":48,"fertility":1.89,"p90":2.38,"cpt":2.49,"neglect":0.2},"qwen2":{"tokens":53,"fertility":2.11,"p90":2.67,"cpt":2.25,"neglect":0.33},"bloom":{"tokens":50,"fertility":2,"p90":2.5,"cpt":2.4,"neglect":0.25}},"massive":{"gpt2":{"tokens":17,"fertility":2.5,"p90":3.5,"cpt":1.99,"neglect":0.55},"cl100k":{"tokens":15,"fertility":2.21,"p90":3.17,"cpt":2.27,"neglect":0.36},"o200k":{"tokens":11,"fertility":1.7,"p90":2.44,"cpt":3,"neglect":0},"llama3":{"tokens":15,"fertility":2.18,"p90":3,"cpt":2.31,"neglect":0.36},"gemma":{"tokens":13,"fertility":2,"p90":2.8,"cpt":2.63,"neglect":0.18},"tekken":{"tokens":14,"fertility":1.93,"p90":2.75,"cpt":2.55,"neglect":0.23},"qwen2":{"tokens":15,"fertility":2.2,"p90":3.12,"cpt":2.3,"neglect":0.36},"bloom":{"tokens":14,"fertility":2,"p90":2.86,"cpt":2.44,"neglect":0.27}}}},{"code":"ita_Latn","name":"Italian","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":64800000,"floor":33,"floors":{"flores":33,"massive":8},"metrics":{"flores":{"gpt2":{"tokens":50,"fertility":1.97,"p90":2.5,"cpt":2.87,"neglect":0.52},"cl100k":{"tokens":41,"fertility":1.62,"p90":2,"cpt":3.48,"neglect":0.24},"o200k":{"tokens":36,"fertility":1.44,"p90":1.78,"cpt":3.96,"neglect":0.09},"llama3":{"tokens":40,"fertility":1.62,"p90":2,"cpt":3.49,"neglect":0.21},"gemma":{"tokens":33,"fertility":1.32,"p90":1.62,"cpt":4.28,"neglect":0},"tekken":{"tokens":35,"fertility":1.38,"p90":1.7,"cpt":4.01,"neglect":0.06},"qwen2":{"tokens":41,"fertility":1.61,"p90":2,"cpt":3.46,"neglect":0.24},"bloom":{"tokens":41,"fertility":1.6,"p90":2,"cpt":3.51,"neglect":0.24}},"massive":{"gpt2":{"tokens":12,"fertility":1.8,"p90":2.6,"cpt":2.85,"neglect":0.5},"cl100k":{"tokens":10,"fertility":1.5,"p90":2.17,"cpt":3.52,"neglect":0.25},"o200k":{"tokens":9,"fertility":1.33,"p90":2,"cpt":3.94,"neglect":0.12},"llama3":{"tokens":10,"fertility":1.5,"p90":2.17,"cpt":3.52,"neglect":0.25},"gemma":{"tokens":8,"fertility":1.2,"p90":1.75,"cpt":4.42,"neglect":0},"tekken":{"tokens":9,"fertility":1.29,"p90":2,"cpt":3.93,"neglect":0.12},"qwen2":{"tokens":10,"fertility":1.5,"p90":2.17,"cpt":3.53,"neglect":0.25},"bloom":{"tokens":10,"fertility":1.5,"p90":2.17,"cpt":3.43,"neglect":0.25}}}},{"code":"jav_Latn","name":"Javanese","script":"Latin","family":"Austronesian","region":"Papunesia","speakers":68300000,"floor":34,"floors":{"flores":34,"massive":9},"metrics":{"flores":{"gpt2":{"tokens":48,"fertility":1.95,"p90":2.42,"cpt":2.62,"neglect":0.41},"cl100k":{"tokens":44,"fertility":1.74,"p90":2.14,"cpt":2.91,"neglect":0.29},"o200k":{"tokens":34,"fertility":1.36,"p90":1.67,"cpt":3.73,"neglect":0},"llama3":{"tokens":43,"fertility":1.73,"p90":2.12,"cpt":2.94,"neglect":0.26},"gemma":{"tokens":37,"fertility":1.48,"p90":1.86,"cpt":3.43,"neglect":0.09},"tekken":{"tokens":42,"fertility":1.64,"p90":2.04,"cpt":3.02,"neglect":0.24},"qwen2":{"tokens":44,"fertility":1.73,"p90":2.13,"cpt":2.88,"neglect":0.29},"bloom":{"tokens":36,"fertility":1.41,"p90":1.75,"cpt":3.55,"neglect":0.06}},"massive":{"gpt2":{"tokens":12,"fertility":1.75,"p90":2.5,"cpt":2.65,"neglect":0.33},"cl100k":{"tokens":11,"fertility":1.57,"p90":2.29,"cpt":3,"neglect":0.22},"o200k":{"tokens":9,"fertility":1.29,"p90":1.86,"cpt":3.71,"neglect":0},"llama3":{"tokens":10,"fertility":1.55,"p90":2.25,"cpt":3.06,"neglect":0.11},"gemma":{"tokens":9,"fertility":1.3,"p90":2,"cpt":3.68,"neglect":0},"tekken":{"tokens":10,"fertility":1.5,"p90":2.17,"cpt":3.08,"neglect":0.11},"qwen2":{"tokens":11,"fertility":1.57,"p90":2.29,"cpt":3,"neglect":0.22},"bloom":{"tokens":9,"fertility":1.25,"p90":1.82,"cpt":3.66,"neglect":0}}}},{"code":"jpn_Jpan","name":"Japanese","script":"Japanese","family":"Japonic","region":"Eurasia","speakers":128000000,"floor":29,"floors":{"flores":29,"massive":7},"metrics":{"flores":{"gpt2":{"tokens":74,"fertility":2.96,"p90":3.73,"cpt":0.72,"neglect":1.55},"cl100k":{"tokens":57,"fertility":2.28,"p90":2.84,"cpt":0.93,"neglect":0.97},"o200k":{"tokens":41,"fertility":1.65,"p90":2.1,"cpt":1.29,"neglect":0.41},"llama3":{"tokens":38,"fertility":1.51,"p90":1.9,"cpt":1.4,"neglect":0.31},"gemma":{"tokens":29,"fertility":1.16,"p90":1.46,"cpt":1.81,"neglect":0},"tekken":{"tokens":40,"fertility":1.59,"p90":2,"cpt":1.31,"neglect":0.38},"qwen2":{"tokens":35,"fertility":1.4,"p90":1.77,"cpt":1.48,"neglect":0.21},"bloom":{"tokens":45,"fertility":1.78,"p90":2.26,"cpt":1.18,"neglect":0.55}},"massive":{"gpt2":{"tokens":19,"fertility":2.8,"p90":4.2,"cpt":0.75,"neglect":1.71},"cl100k":{"tokens":14,"fertility":2.13,"p90":3.25,"cpt":1,"neglect":1},"o200k":{"tokens":10,"fertility":1.56,"p90":2.4,"cpt":1.39,"neglect":0.43},"llama3":{"tokens":10,"fertility":1.5,"p90":2.25,"cpt":1.46,"neglect":0.43},"gemma":{"tokens":7,"fertility":1,"p90":1.5,"cpt":2.11,"neglect":0},"tekken":{"tokens":10,"fertility":1.44,"p90":2.25,"cpt":1.43,"neglect":0.43},"qwen2":{"tokens":9,"fertility":1.33,"p90":2,"cpt":1.6,"neglect":0.29},"bloom":{"tokens":12,"fertility":1.75,"p90":2.71,"cpt":1.19,"neglect":0.71}}}},{"code":"kab_Latn","name":"Kabyle","script":"Latin","family":"Afro-Asiatic","region":"Africa","speakers":5586000,"floor":50,"floors":{"flores":50},"metrics":{"flores":{"gpt2":{"tokens":62,"fertility":2.52,"p90":3.14,"cpt":1.94,"neglect":0.24},"cl100k":{"tokens":62,"fertility":2.44,"p90":3.16,"cpt":1.95,"neglect":0.24},"o200k":{"tokens":53,"fertility":2.14,"p90":2.68,"cpt":2.27,"neglect":0.06},"llama3":{"tokens":60,"fertility":2.37,"p90":3.03,"cpt":2.02,"neglect":0.2},"gemma":{"tokens":51,"fertility":2.05,"p90":2.57,"cpt":2.34,"neglect":0.02},"tekken":{"tokens":59,"fertility":2.28,"p90":2.91,"cpt":2.04,"neglect":0.18},"qwen2":{"tokens":54,"fertility":2.12,"p90":2.67,"cpt":2.22,"neglect":0.08},"bloom":{"tokens":50,"fertility":2.04,"p90":2.5,"cpt":2.37,"neglect":0}}}},{"code":"kac_Latn","name":"Southern Jinghpaw","script":"Latin","family":"Sino-Tibetan","region":"Eurasia","speakers":940000,"floor":50,"floors":{"flores":50},"metrics":{"flores":{"gpt2":{"tokens":64,"fertility":2.6,"p90":3.61,"cpt":2.36,"neglect":0.28},"cl100k":{"tokens":57,"fertility":2.3,"p90":3.16,"cpt":2.65,"neglect":0.14},"o200k":{"tokens":50,"fertility":2.06,"p90":2.83,"cpt":3,"neglect":0},"llama3":{"tokens":56,"fertility":2.27,"p90":3.12,"cpt":2.68,"neglect":0.12},"gemma":{"tokens":51,"fertility":2.05,"p90":2.84,"cpt":2.97,"neglect":0.02},"tekken":{"tokens":56,"fertility":2.19,"p90":3,"cpt":2.71,"neglect":0.12},"qwen2":{"tokens":57,"fertility":2.27,"p90":3.14,"cpt":2.64,"neglect":0.14},"bloom":{"tokens":53,"fertility":2.11,"p90":2.88,"cpt":2.87,"neglect":0.06}}}},{"code":"kam_Latn","name":"Kamba (Kenya)","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":3893000,"floor":43,"floors":{"flores":43},"metrics":{"flores":{"gpt2":{"tokens":54,"fertility":2.15,"p90":3.45,"cpt":2.06,"neglect":0.26},"cl100k":{"tokens":51,"fertility":2,"p90":3.28,"cpt":2.19,"neglect":0.19},"o200k":{"tokens":43,"fertility":1.73,"p90":2.59,"cpt":2.64,"neglect":0},"llama3":{"tokens":49,"fertility":1.94,"p90":2.76,"cpt":2.37,"neglect":0.14},"gemma":{"tokens":44,"fertility":1.75,"p90":2.56,"cpt":2.6,"neglect":0.02},"tekken":{"tokens":48,"fertility":1.86,"p90":2.66,"cpt":2.4,"neglect":0.12},"qwen2":{"tokens":49,"fertility":1.93,"p90":2.72,"cpt":2.36,"neglect":0.14},"bloom":{"tokens":43,"fertility":1.68,"p90":2.47,"cpt":2.68,"neglect":0}}}},{"code":"kan_Knda","name":"Kannada","script":"Kannada","family":"Dravidian","region":"Eurasia","speakers":43600000,"floor":33,"floors":{"flores":33,"massive":9},"metrics":{"flores":{"gpt2":{"tokens":336,"fertility":13.62,"p90":17.16,"cpt":0.38,"neglect":9.18},"cl100k":{"tokens":221,"fertility":8.89,"p90":11.25,"cpt":0.58,"neglect":5.7},"o200k":{"tokens":49,"fertility":2,"p90":2.5,"cpt":2.58,"neglect":0.48},"llama3":{"tokens":221,"fertility":8.89,"p90":11.25,"cpt":0.58,"neglect":5.7},"gemma":{"tokens":83,"fertility":3.34,"p90":4.14,"cpt":1.53,"neglect":1.52},"tekken":{"tokens":57,"fertility":2.23,"p90":2.8,"cpt":2.22,"neglect":0.73},"qwen2":{"tokens":175,"fertility":6.96,"p90":8.84,"cpt":0.72,"neglect":4.3},"bloom":{"tokens":33,"fertility":1.31,"p90":1.68,"cpt":3.8,"neglect":0}},"massive":{"gpt2":{"tokens":96,"fertility":13.85,"p90":20,"cpt":0.37,"neglect":9.67},"cl100k":{"tokens":62,"fertility":9.2,"p90":13.33,"cpt":0.57,"neglect":5.89},"o200k":{"tokens":14,"fertility":2,"p90":3,"cpt":2.57,"neglect":0.56},"llama3":{"tokens":62,"fertility":9.2,"p90":13.33,"cpt":0.57,"neglect":5.89},"gemma":{"tokens":23,"fertility":3.45,"p90":5,"cpt":1.54,"neglect":1.56},"tekken":{"tokens":17,"fertility":2.44,"p90":3.6,"cpt":2.07,"neglect":0.89},"qwen2":{"tokens":50,"fertility":7.29,"p90":10.5,"cpt":0.72,"neglect":4.56},"bloom":{"tokens":9,"fertility":1.36,"p90":2,"cpt":3.75,"neglect":0}}}},{"code":"kas_Arab","name":"Kashmiri","script":"Arabic","family":"Indo-European","region":"Eurasia","speakers":6900000,"floor":56,"floors":{"flores":56},"metrics":{"flores":{"gpt2":{"tokens":152,"fertility":6.11,"p90":7.64,"cpt":0.76,"neglect":1.71},"cl100k":{"tokens":115,"fertility":4.61,"p90":5.83,"cpt":1.01,"neglect":1.05},"o200k":{"tokens":63,"fertility":2.52,"p90":3.38,"cpt":1.83,"neglect":0.12},"llama3":{"tokens":80,"fertility":3.21,"p90":4.21,"cpt":1.43,"neglect":0.43},"gemma":{"tokens":63,"fertility":2.55,"p90":3.21,"cpt":1.82,"neglect":0.12},"tekken":{"tokens":65,"fertility":2.57,"p90":3.33,"cpt":1.76,"neglect":0.16},"qwen2":{"tokens":87,"fertility":3.45,"p90":4.43,"cpt":1.33,"neglect":0.55},"bloom":{"tokens":56,"fertility":2.26,"p90":3.05,"cpt":2.02,"neglect":0}}}},{"code":"kas_Deva","name":"Kashmiri","script":"Devanagari","family":"Indo-European","region":"Eurasia","speakers":6900000,"floor":46,"floors":{"flores":46},"metrics":{"flores":{"gpt2":{"tokens":176,"fertility":6.97,"p90":8.86,"cpt":0.67,"neglect":2.83},"cl100k":{"tokens":117,"fertility":4.69,"p90":5.89,"cpt":1,"neglect":1.54},"o200k":{"tokens":54,"fertility":2.2,"p90":2.78,"cpt":2.12,"neglect":0.17},"llama3":{"tokens":66,"fertility":2.64,"p90":3.31,"cpt":1.76,"neglect":0.43},"gemma":{"tokens":61,"fertility":2.47,"p90":3.16,"cpt":1.88,"neglect":0.33},"tekken":{"tokens":62,"fertility":2.44,"p90":3.11,"cpt":1.86,"neglect":0.35},"qwen2":{"tokens":112,"fertility":4.4,"p90":5.54,"cpt":1.05,"neglect":1.43},"bloom":{"tokens":46,"fertility":1.84,"p90":2.33,"cpt":2.5,"neglect":0}}}},{"code":"kat_Geor","name":"Georgian","script":"Georgian","family":"Kartvelian","region":"Eurasia","speakers":3700000,"floor":44,"floors":{"flores":44,"massive":11},"metrics":{"flores":{"gpt2":{"tokens":342,"fertility":13.8,"p90":18.18,"cpt":0.39,"neglect":6.77},"cl100k":{"tokens":244,"fertility":9.8,"p90":12.79,"cpt":0.54,"neglect":4.55},"o200k":{"tokens":44,"fertility":1.77,"p90":2.25,"cpt":3.01,"neglect":0},"llama3":{"tokens":244,"fertility":9.8,"p90":12.79,"cpt":0.54,"neglect":4.55},"gemma":{"tokens":92,"fertility":3.69,"p90":4.76,"cpt":1.45,"neglect":1.09},"tekken":{"tokens":61,"fertility":2.4,"p90":3.09,"cpt":2.16,"neglect":0.39},"qwen2":{"tokens":132,"fertility":5.21,"p90":6.84,"cpt":1.01,"neglect":2},"bloom":{"tokens":125,"fertility":4.96,"p90":6.36,"cpt":1.07,"neglect":1.84}},"massive":{"gpt2":{"tokens":75,"fertility":11.4,"p90":18,"cpt":0.38,"neglect":5.82},"cl100k":{"tokens":53,"fertility":8.25,"p90":12.71,"cpt":0.53,"neglect":3.82},"o200k":{"tokens":11,"fertility":1.62,"p90":2.67,"cpt":2.74,"neglect":0},"llama3":{"tokens":53,"fertility":8.25,"p90":12.71,"cpt":0.53,"neglect":3.82},"gemma":{"tokens":20,"fertility":3.14,"p90":4.86,"cpt":1.42,"neglect":0.82},"tekken":{"tokens":14,"fertility":2.14,"p90":3.33,"cpt":2,"neglect":0.27},"qwen2":{"tokens":28,"fertility":4.38,"p90":6.67,"cpt":1.01,"neglect":1.55},"bloom":{"tokens":27,"fertility":4.12,"p90":6.33,"cpt":1.04,"neglect":1.45}}}},{"code":"kaz_Cyrl","name":"Kazakh","script":"Cyrillic","family":"Turkic","region":"Eurasia","speakers":12900000,"floor":39,"floors":{"flores":39},"metrics":{"flores":{"gpt2":{"tokens":145,"fertility":5.87,"p90":7.52,"cpt":0.85,"neglect":2.72},"cl100k":{"tokens":93,"fertility":3.77,"p90":4.83,"cpt":1.32,"neglect":1.38},"o200k":{"tokens":39,"fertility":1.57,"p90":1.95,"cpt":3.18,"neglect":0},"llama3":{"tokens":76,"fertility":3.06,"p90":3.92,"cpt":1.62,"neglect":0.95},"gemma":{"tokens":58,"fertility":2.3,"p90":3,"cpt":2.15,"neglect":0.49},"tekken":{"tokens":52,"fertility":2.04,"p90":2.6,"cpt":2.37,"neglect":0.33},"qwen2":{"tokens":76,"fertility":3,"p90":3.86,"cpt":1.63,"neglect":0.95},"bloom":{"tokens":80,"fertility":3.21,"p90":4.07,"cpt":1.54,"neglect":1.05}}}},{"code":"kbp_Latn","name":"Kabiyé","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":1000000,"floor":80,"floors":{"flores":80},"metrics":{"flores":{"gpt2":{"tokens":120,"fertility":4.81,"p90":6.78,"cpt":1.09,"neglect":0.5},"cl100k":{"tokens":118,"fertility":4.68,"p90":6.7,"cpt":1.12,"neglect":0.47},"o200k":{"tokens":96,"fertility":3.86,"p90":5.47,"cpt":1.36,"neglect":0.2},"llama3":{"tokens":117,"fertility":4.68,"p90":6.67,"cpt":1.12,"neglect":0.46},"gemma":{"tokens":80,"fertility":3.21,"p90":4.5,"cpt":1.63,"neglect":0},"tekken":{"tokens":115,"fertility":4.48,"p90":6.37,"cpt":1.14,"neglect":0.44},"qwen2":{"tokens":83,"fertility":3.29,"p90":4.64,"cpt":1.56,"neglect":0.04},"bloom":{"tokens":83,"fertility":3.31,"p90":4.55,"cpt":1.57,"neglect":0.04}}}},{"code":"kea_Latn","name":"Kabuverdianu","script":"Latin","family":"Indo-European","region":"Africa","speakers":871000,"floor":38,"floors":{"flores":38},"metrics":{"flores":{"gpt2":{"tokens":48,"fertility":1.94,"p90":2.35,"cpt":2.5,"neglect":0.26},"cl100k":{"tokens":43,"fertility":1.71,"p90":2.09,"cpt":2.8,"neglect":0.13},"o200k":{"tokens":38,"fertility":1.54,"p90":1.85,"cpt":3.16,"neglect":0},"llama3":{"tokens":42,"fertility":1.68,"p90":2.04,"cpt":2.86,"neglect":0.11},"gemma":{"tokens":38,"fertility":1.53,"p90":1.87,"cpt":3.15,"neglect":0},"tekken":{"tokens":40,"fertility":1.55,"p90":1.88,"cpt":3.02,"neglect":0.05},"qwen2":{"tokens":43,"fertility":1.69,"p90":2.06,"cpt":2.79,"neglect":0.13},"bloom":{"tokens":38,"fertility":1.5,"p90":1.8,"cpt":3.18,"neglect":0}}}},{"code":"khk_Cyrl","name":"Halh Mongolian","script":"Cyrillic","family":"Mongolic-Khitan","region":"Eurasia","speakers":2800000,"floor":46,"floors":{"flores":46,"massive":13},"metrics":{"flores":{"gpt2":{"tokens":159,"fertility":6.42,"p90":8.1,"cpt":0.8,"neglect":2.46},"cl100k":{"tokens":93,"fertility":3.73,"p90":4.83,"cpt":1.36,"neglect":1.02},"o200k":{"tokens":46,"fertility":1.88,"p90":2.33,"cpt":2.73,"neglect":0},"llama3":{"tokens":80,"fertility":3.23,"p90":4.21,"cpt":1.58,"neglect":0.74},"gemma":{"tokens":66,"fertility":2.65,"p90":3.41,"cpt":1.93,"neglect":0.43},"tekken":{"tokens":61,"fertility":2.38,"p90":3,"cpt":2.09,"neglect":0.33},"qwen2":{"tokens":78,"fertility":3.1,"p90":4,"cpt":1.62,"neglect":0.7},"bloom":{"tokens":86,"fertility":3.4,"p90":4.29,"cpt":1.49,"neglect":0.87}},"massive":{"gpt2":{"tokens":45,"fertility":6.33,"p90":9.43,"cpt":0.79,"neglect":2.46},"cl100k":{"tokens":26,"fertility":3.8,"p90":5.83,"cpt":1.35,"neglect":1},"o200k":{"tokens":13,"fertility":1.86,"p90":2.8,"cpt":2.81,"neglect":0},"llama3":{"tokens":22,"fertility":3.25,"p90":5.12,"cpt":1.58,"neglect":0.69},"gemma":{"tokens":18,"fertility":2.65,"p90":4,"cpt":1.98,"neglect":0.38},"tekken":{"tokens":17,"fertility":2.4,"p90":3.5,"cpt":2.09,"neglect":0.31},"qwen2":{"tokens":22,"fertility":3.12,"p90":4.75,"cpt":1.64,"neglect":0.69},"bloom":{"tokens":23,"fertility":3.33,"p90":5,"cpt":1.52,"neglect":0.77}}}},{"code":"khm_Khmr","name":"Central Khmer","script":"Khmer","family":"Austroasiatic","region":"Eurasia","speakers":16600000,"floor":82,"floors":{"flores":82,"massive":17},"metrics":{"flores":{"gpt2":{"tokens":379,"fertility":15.33,"p90":19.79,"cpt":0.38,"neglect":3.62},"cl100k":{"tokens":220,"fertility":8.91,"p90":11.5,"cpt":0.65,"neglect":1.68},"o200k":{"tokens":82,"fertility":3.33,"p90":4.25,"cpt":1.76,"neglect":0},"llama3":{"tokens":220,"fertility":8.91,"p90":11.5,"cpt":0.65,"neglect":1.68},"gemma":{"tokens":136,"fertility":5.48,"p90":7.16,"cpt":1.06,"neglect":0.66},"tekken":{"tokens":378,"fertility":15.03,"p90":19.28,"cpt":0.38,"neglect":3.61},"qwen2":{"tokens":167,"fertility":6.67,"p90":8.61,"cpt":0.86,"neglect":1.04},"bloom":{"tokens":161,"fertility":6.43,"p90":8.33,"cpt":0.9,"neglect":0.96}},"massive":{"gpt2":{"tokens":78,"fertility":12,"p90":19.15,"cpt":0.37,"neglect":3.59},"cl100k":{"tokens":46,"fertility":7.33,"p90":11.5,"cpt":0.62,"neglect":1.71},"o200k":{"tokens":17,"fertility":2.71,"p90":4.2,"cpt":1.7,"neglect":0},"llama3":{"tokens":46,"fertility":7.33,"p90":11.5,"cpt":0.62,"neglect":1.71},"gemma":{"tokens":27,"fertility":4.44,"p90":7,"cpt":1.03,"neglect":0.59},"tekken":{"tokens":77,"fertility":11.91,"p90":19,"cpt":0.37,"neglect":3.53},"qwen2":{"tokens":35,"fertility":5.6,"p90":8.75,"cpt":0.81,"neglect":1.06},"bloom":{"tokens":32,"fertility":5,"p90":7.88,"cpt":0.88,"neglect":0.88}}}},{"code":"kik_Latn","name":"Kikuyu","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":6623000,"floor":62,"floors":{"flores":62},"metrics":{"flores":{"gpt2":{"tokens":85,"fertility":3.42,"p90":4.67,"cpt":1.65,"neglect":0.37},"cl100k":{"tokens":82,"fertility":3.28,"p90":4.5,"cpt":1.72,"neglect":0.32},"o200k":{"tokens":65,"fertility":2.61,"p90":3.51,"cpt":2.18,"neglect":0.05},"llama3":{"tokens":67,"fertility":2.71,"p90":3.59,"cpt":2.1,"neglect":0.08},"gemma":{"tokens":63,"fertility":2.52,"p90":3.36,"cpt":2.25,"neglect":0.02},"tekken":{"tokens":67,"fertility":2.61,"p90":3.47,"cpt":2.11,"neglect":0.08},"qwen2":{"tokens":67,"fertility":2.65,"p90":3.54,"cpt":2.1,"neglect":0.08},"bloom":{"tokens":62,"fertility":2.49,"p90":3.28,"cpt":2.27,"neglect":0}}}},{"code":"kin_Latn","name":"Kinyarwanda","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":12100000,"floor":40,"floors":{"flores":40},"metrics":{"flores":{"gpt2":{"tokens":59,"fertility":2.35,"p90":3,"cpt":2.3,"neglect":0.47},"cl100k":{"tokens":53,"fertility":2.14,"p90":2.7,"cpt":2.53,"neglect":0.33},"o200k":{"tokens":42,"fertility":1.71,"p90":2.11,"cpt":3.19,"neglect":0.05},"llama3":{"tokens":53,"fertility":2.12,"p90":2.67,"cpt":2.55,"neglect":0.33},"gemma":{"tokens":49,"fertility":2,"p90":2.52,"cpt":2.72,"neglect":0.23},"tekken":{"tokens":52,"fertility":2.04,"p90":2.58,"cpt":2.6,"neglect":0.3},"qwen2":{"tokens":53,"fertility":2.12,"p90":2.67,"cpt":2.52,"neglect":0.33},"bloom":{"tokens":40,"fertility":1.57,"p90":1.95,"cpt":3.4,"neglect":0}}}},{"code":"kir_Cyrl","name":"Kirghiz","script":"Cyrillic","family":"Turkic","region":"Eurasia","speakers":4568480,"floor":41,"floors":{"flores":41},"metrics":{"flores":{"gpt2":{"tokens":141,"fertility":5.59,"p90":7.51,"cpt":0.87,"neglect":2.44},"cl100k":{"tokens":86,"fertility":3.45,"p90":4.68,"cpt":1.42,"neglect":1.1},"o200k":{"tokens":41,"fertility":1.64,"p90":2.11,"cpt":3.03,"neglect":0},"llama3":{"tokens":71,"fertility":2.85,"p90":3.91,"cpt":1.72,"neglect":0.73},"gemma":{"tokens":58,"fertility":2.3,"p90":3.16,"cpt":2.12,"neglect":0.41},"tekken":{"tokens":56,"fertility":2.14,"p90":2.89,"cpt":2.23,"neglect":0.37},"qwen2":{"tokens":69,"fertility":2.7,"p90":3.63,"cpt":1.79,"neglect":0.68},"bloom":{"tokens":76,"fertility":3,"p90":3.97,"cpt":1.64,"neglect":0.85}}}},{"code":"kmb_Latn","name":"Kimbundu","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":2000000,"floor":45,"floors":{"flores":45},"metrics":{"flores":{"gpt2":{"tokens":58,"fertility":2.32,"p90":3,"cpt":2.32,"neglect":0.29},"cl100k":{"tokens":53,"fertility":2.12,"p90":2.73,"cpt":2.53,"neglect":0.18},"o200k":{"tokens":45,"fertility":1.82,"p90":2.33,"cpt":2.98,"neglect":0},"llama3":{"tokens":52,"fertility":2.1,"p90":2.71,"cpt":2.56,"neglect":0.16},"gemma":{"tokens":47,"fertility":1.86,"p90":2.41,"cpt":2.87,"neglect":0.04},"tekken":{"tokens":50,"fertility":1.97,"p90":2.55,"cpt":2.65,"neglect":0.11},"qwen2":{"tokens":53,"fertility":2.09,"p90":2.72,"cpt":2.52,"neglect":0.18},"bloom":{"tokens":45,"fertility":1.81,"p90":2.3,"cpt":2.95,"neglect":0}}}},{"code":"kmr_Latn","name":"Northern Kurdish","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":14600000,"floor":49,"floors":{"flores":49},"metrics":{"flores":{"gpt2":{"tokens":61,"fertility":2.43,"p90":3.11,"cpt":1.99,"neglect":0.24},"cl100k":{"tokens":55,"fertility":2.2,"p90":2.75,"cpt":2.2,"neglect":0.12},"o200k":{"tokens":50,"fertility":2,"p90":2.53,"cpt":2.43,"neglect":0.02},"llama3":{"tokens":53,"fertility":2.16,"p90":2.7,"cpt":2.25,"neglect":0.08},"gemma":{"tokens":49,"fertility":2,"p90":2.49,"cpt":2.43,"neglect":0},"tekken":{"tokens":51,"fertility":2.04,"p90":2.53,"cpt":2.34,"neglect":0.04},"qwen2":{"tokens":55,"fertility":2.17,"p90":2.71,"cpt":2.21,"neglect":0.12},"bloom":{"tokens":51,"fertility":2.03,"p90":2.54,"cpt":2.38,"neglect":0.04}}}},{"code":"knc_Arab","name":"Central Kanuri","script":"Arabic","family":"Saharan","region":"Africa","speakers":9270000,"floor":47,"floors":{"flores":47},"metrics":{"flores":{"gpt2":{"tokens":108,"fertility":4.58,"p90":7.22,"cpt":0.91,"neglect":1.3},"cl100k":{"tokens":82,"fertility":3.59,"p90":5.68,"cpt":1.18,"neglect":0.74},"o200k":{"tokens":58,"fertility":2.73,"p90":4.57,"cpt":1.59,"neglect":0.23},"llama3":{"tokens":58,"fertility":2.59,"p90":4.24,"cpt":1.63,"neglect":0.23},"gemma":{"tokens":59,"fertility":2.65,"p90":4.6,"cpt":1.57,"neglect":0.26},"tekken":{"tokens":55,"fertility":2.42,"p90":4.09,"cpt":1.69,"neglect":0.17},"qwen2":{"tokens":68,"fertility":3.12,"p90":5.31,"cpt":1.35,"neglect":0.45},"bloom":{"tokens":47,"fertility":2.06,"p90":3.31,"cpt":2.03,"neglect":0}}}},{"code":"knc_Latn","name":"Central Kanuri","script":"Latin","family":"Saharan","region":"Africa","speakers":9270000,"floor":48,"floors":{"flores":48},"metrics":{"flores":{"gpt2":{"tokens":62,"fertility":2.54,"p90":3.42,"cpt":2,"neglect":0.29},"cl100k":{"tokens":57,"fertility":2.31,"p90":3.12,"cpt":2.17,"neglect":0.19},"o200k":{"tokens":51,"fertility":2.06,"p90":2.82,"cpt":2.45,"neglect":0.06},"llama3":{"tokens":56,"fertility":2.28,"p90":3.04,"cpt":2.21,"neglect":0.17},"gemma":{"tokens":48,"fertility":1.95,"p90":2.58,"cpt":2.6,"neglect":0},"tekken":{"tokens":54,"fertility":2.14,"p90":2.91,"cpt":2.29,"neglect":0.12},"qwen2":{"tokens":53,"fertility":2.14,"p90":2.83,"cpt":2.33,"neglect":0.1},"bloom":{"tokens":49,"fertility":1.96,"p90":2.63,"cpt":2.55,"neglect":0.02}}}},{"code":"kon_Latn","name":"Kikongo","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":5000000,"floor":43,"floors":{"flores":43},"metrics":{"flores":{"gpt2":{"tokens":54,"fertility":2.16,"p90":2.79,"cpt":2.56,"neglect":0.26},"cl100k":{"tokens":50,"fertility":2,"p90":2.54,"cpt":2.78,"neglect":0.16},"o200k":{"tokens":43,"fertility":1.75,"p90":2.21,"cpt":3.18,"neglect":0},"llama3":{"tokens":49,"fertility":1.97,"p90":2.52,"cpt":2.81,"neglect":0.14},"gemma":{"tokens":45,"fertility":1.79,"p90":2.28,"cpt":3.1,"neglect":0.05},"tekken":{"tokens":48,"fertility":1.89,"p90":2.44,"cpt":2.85,"neglect":0.12},"qwen2":{"tokens":50,"fertility":2,"p90":2.54,"cpt":2.76,"neglect":0.16},"bloom":{"tokens":44,"fertility":1.75,"p90":2.19,"cpt":3.15,"neglect":0.02}}}},{"code":"kor_Hang","name":"Korean","script":"Hangul","family":"Koreanic","region":"Eurasia","speakers":77300000,"floor":32,"floors":{"flores":32,"massive":9},"metrics":{"flores":{"gpt2":{"tokens":127,"fertility":5.04,"p90":6.32,"cpt":0.48,"neglect":2.97},"cl100k":{"tokens":59,"fertility":2.33,"p90":2.96,"cpt":1.03,"neglect":0.84},"o200k":{"tokens":36,"fertility":1.46,"p90":1.83,"cpt":1.68,"neglect":0.12},"llama3":{"tokens":37,"fertility":1.48,"p90":1.85,"cpt":1.64,"neglect":0.16},"gemma":{"tokens":40,"fertility":1.63,"p90":2.04,"cpt":1.49,"neglect":0.25},"tekken":{"tokens":32,"fertility":1.27,"p90":1.61,"cpt":1.85,"neglect":0},"qwen2":{"tokens":41,"fertility":1.62,"p90":2.05,"cpt":1.46,"neglect":0.28},"bloom":{"tokens":70,"fertility":2.75,"p90":3.44,"cpt":0.87,"neglect":1.19}},"massive":{"gpt2":{"tokens":31,"fertility":4.5,"p90":6.57,"cpt":0.46,"neglect":2.44},"cl100k":{"tokens":15,"fertility":2.23,"p90":3.33,"cpt":0.95,"neglect":0.67},"o200k":{"tokens":9,"fertility":1.33,"p90":2,"cpt":1.58,"neglect":0},"llama3":{"tokens":10,"fertility":1.4,"p90":2.14,"cpt":1.49,"neglect":0.11},"gemma":{"tokens":10,"fertility":1.5,"p90":2.2,"cpt":1.41,"neglect":0.11},"tekken":{"tokens":9,"fertility":1.29,"p90":2,"cpt":1.6,"neglect":0},"qwen2":{"tokens":11,"fertility":1.57,"p90":2.33,"cpt":1.35,"neglect":0.22},"bloom":{"tokens":18,"fertility":2.67,"p90":3.88,"cpt":0.78,"neglect":1}}}},{"code":"lao_Laoo","name":"Lao","script":"Lao","family":"Tai-Kadai","region":"Eurasia","speakers":5225552,"floor":114,"floors":{"flores":114},"metrics":{"flores":{"gpt2":{"tokens":323,"fertility":13.18,"p90":17,"cpt":0.37,"neglect":1.83},"cl100k":{"tokens":237,"fertility":9.61,"p90":12.58,"cpt":0.51,"neglect":1.08},"o200k":{"tokens":203,"fertility":8.24,"p90":10.76,"cpt":0.59,"neglect":0.78},"llama3":{"tokens":219,"fertility":8.85,"p90":11.44,"cpt":0.55,"neglect":0.92},"gemma":{"tokens":114,"fertility":4.58,"p90":5.98,"cpt":1.06,"neglect":0},"tekken":{"tokens":323,"fertility":12.73,"p90":16.57,"cpt":0.37,"neglect":1.83},"qwen2":{"tokens":144,"fertility":5.75,"p90":7.45,"cpt":0.83,"neglect":0.26},"bloom":{"tokens":216,"fertility":8.68,"p90":11.14,"cpt":0.56,"neglect":0.89}}}},{"code":"lij_Latn","name":"Ligurian","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":500000,"floor":44,"floors":{"flores":44},"metrics":{"flores":{"gpt2":{"tokens":57,"fertility":2.26,"p90":2.85,"cpt":2.36,"neglect":0.3},"cl100k":{"tokens":49,"fertility":1.95,"p90":2.44,"cpt":2.71,"neglect":0.11},"o200k":{"tokens":44,"fertility":1.79,"p90":2.2,"cpt":3,"neglect":0},"llama3":{"tokens":49,"fertility":1.94,"p90":2.41,"cpt":2.73,"neglect":0.11},"gemma":{"tokens":45,"fertility":1.83,"p90":2.26,"cpt":2.92,"neglect":0.02},"tekken":{"tokens":46,"fertility":1.81,"p90":2.21,"cpt":2.88,"neglect":0.05},"qwen2":{"tokens":49,"fertility":1.94,"p90":2.42,"cpt":2.7,"neglect":0.11},"bloom":{"tokens":45,"fertility":1.8,"p90":2.19,"cpt":2.95,"neglect":0.02}}}},{"code":"lim_Latn","name":"Limburgan","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":1600000,"floor":39,"floors":{"flores":39},"metrics":{"flores":{"gpt2":{"tokens":50,"fertility":2.03,"p90":2.53,"cpt":2.48,"neglect":0.28},"cl100k":{"tokens":44,"fertility":1.78,"p90":2.21,"cpt":2.81,"neglect":0.13},"o200k":{"tokens":39,"fertility":1.57,"p90":1.95,"cpt":3.21,"neglect":0},"llama3":{"tokens":44,"fertility":1.77,"p90":2.19,"cpt":2.83,"neglect":0.13},"gemma":{"tokens":39,"fertility":1.59,"p90":1.96,"cpt":3.15,"neglect":0},"tekken":{"tokens":43,"fertility":1.67,"p90":2.05,"cpt":2.94,"neglect":0.1},"qwen2":{"tokens":45,"fertility":1.77,"p90":2.2,"cpt":2.79,"neglect":0.15},"bloom":{"tokens":43,"fertility":1.74,"p90":2.14,"cpt":2.87,"neglect":0.1}}}},{"code":"lin_Latn","name":"Kinshasa Lingala","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":20000000,"floor":40,"floors":{"flores":40},"metrics":{"flores":{"gpt2":{"tokens":50,"fertility":2,"p90":2.52,"cpt":2.6,"neglect":0.25},"cl100k":{"tokens":46,"fertility":1.85,"p90":2.3,"cpt":2.81,"neglect":0.15},"o200k":{"tokens":40,"fertility":1.59,"p90":2,"cpt":3.28,"neglect":0},"llama3":{"tokens":46,"fertility":1.83,"p90":2.29,"cpt":2.83,"neglect":0.15},"gemma":{"tokens":43,"fertility":1.71,"p90":2.17,"cpt":3.03,"neglect":0.07},"tekken":{"tokens":45,"fertility":1.77,"p90":2.21,"cpt":2.87,"neglect":0.12},"qwen2":{"tokens":46,"fertility":1.84,"p90":2.3,"cpt":2.79,"neglect":0.15},"bloom":{"tokens":41,"fertility":1.63,"p90":2,"cpt":3.16,"neglect":0.03}}}},{"code":"lit_Latn","name":"Lithuanian","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":4000000,"floor":40,"floors":{"flores":40},"metrics":{"flores":{"gpt2":{"tokens":61,"fertility":2.44,"p90":3.1,"cpt":1.99,"neglect":0.53},"cl100k":{"tokens":55,"fertility":2.21,"p90":2.81,"cpt":2.19,"neglect":0.38},"o200k":{"tokens":42,"fertility":1.7,"p90":2.12,"cpt":2.89,"neglect":0.05},"llama3":{"tokens":54,"fertility":2.17,"p90":2.75,"cpt":2.24,"neglect":0.35},"gemma":{"tokens":40,"fertility":1.62,"p90":2.05,"cpt":2.98,"neglect":0},"tekken":{"tokens":45,"fertility":1.75,"p90":2.21,"cpt":2.7,"neglect":0.12},"qwen2":{"tokens":52,"fertility":2.06,"p90":2.62,"cpt":2.31,"neglect":0.3},"bloom":{"tokens":49,"fertility":1.96,"p90":2.44,"cpt":2.47,"neglect":0.23}}}},{"code":"lmo_Latn","name":"Lombard","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":3900000,"floor":45,"floors":{"flores":45},"metrics":{"flores":{"gpt2":{"tokens":59,"fertility":2.36,"p90":3,"cpt":2.22,"neglect":0.31},"cl100k":{"tokens":51,"fertility":2.04,"p90":2.56,"cpt":2.57,"neglect":0.13},"o200k":{"tokens":45,"fertility":1.83,"p90":2.29,"cpt":2.88,"neglect":0},"llama3":{"tokens":50,"fertility":2,"p90":2.52,"cpt":2.6,"neglect":0.11},"gemma":{"tokens":46,"fertility":1.86,"p90":2.31,"cpt":2.81,"neglect":0.02},"tekken":{"tokens":47,"fertility":1.87,"p90":2.35,"cpt":2.72,"neglect":0.04},"qwen2":{"tokens":50,"fertility":2,"p90":2.51,"cpt":2.57,"neglect":0.11},"bloom":{"tokens":46,"fertility":1.83,"p90":2.25,"cpt":2.83,"neglect":0.02}}}},{"code":"ltg_Latn","name":"Latgalian","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":150000,"floor":46,"floors":{"flores":46},"metrics":{"flores":{"gpt2":{"tokens":59,"fertility":2.4,"p90":3,"cpt":2.02,"neglect":0.28},"cl100k":{"tokens":55,"fertility":2.2,"p90":2.75,"cpt":2.18,"neglect":0.2},"o200k":{"tokens":46,"fertility":1.89,"p90":2.37,"cpt":2.56,"neglect":0},"llama3":{"tokens":53,"fertility":2.16,"p90":2.69,"cpt":2.23,"neglect":0.15},"gemma":{"tokens":46,"fertility":1.86,"p90":2.32,"cpt":2.59,"neglect":0},"tekken":{"tokens":49,"fertility":1.94,"p90":2.41,"cpt":2.42,"neglect":0.07},"qwen2":{"tokens":54,"fertility":2.14,"p90":2.69,"cpt":2.2,"neglect":0.17},"bloom":{"tokens":50,"fertility":2,"p90":2.45,"cpt":2.39,"neglect":0.09}}}},{"code":"ltz_Latn","name":"Moselle Franconian","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":391200,"floor":40,"floors":{"flores":40},"metrics":{"flores":{"gpt2":{"tokens":55,"fertility":2.23,"p90":2.83,"cpt":2.44,"neglect":0.38},"cl100k":{"tokens":49,"fertility":1.96,"p90":2.46,"cpt":2.75,"neglect":0.23},"o200k":{"tokens":40,"fertility":1.67,"p90":2.06,"cpt":3.3,"neglect":0},"llama3":{"tokens":49,"fertility":1.96,"p90":2.45,"cpt":2.77,"neglect":0.23},"gemma":{"tokens":43,"fertility":1.76,"p90":2.21,"cpt":3.07,"neglect":0.07},"tekken":{"tokens":46,"fertility":1.84,"p90":2.29,"cpt":2.88,"neglect":0.15},"qwen2":{"tokens":49,"fertility":1.95,"p90":2.43,"cpt":2.73,"neglect":0.23},"bloom":{"tokens":47,"fertility":1.89,"p90":2.32,"cpt":2.86,"neglect":0.17}}}},{"code":"lua_Latn","name":"Luba-Lulua","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":6300000,"floor":41,"floors":{"flores":41},"metrics":{"flores":{"gpt2":{"tokens":53,"fertility":2.1,"p90":2.71,"cpt":2.48,"neglect":0.29},"cl100k":{"tokens":48,"fertility":1.92,"p90":2.46,"cpt":2.7,"neglect":0.17},"o200k":{"tokens":41,"fertility":1.67,"p90":2.11,"cpt":3.15,"neglect":0},"llama3":{"tokens":47,"fertility":1.89,"p90":2.41,"cpt":2.74,"neglect":0.15},"gemma":{"tokens":44,"fertility":1.74,"p90":2.23,"cpt":2.98,"neglect":0.07},"tekken":{"tokens":46,"fertility":1.81,"p90":2.33,"cpt":2.79,"neglect":0.12},"qwen2":{"tokens":48,"fertility":1.9,"p90":2.44,"cpt":2.68,"neglect":0.17},"bloom":{"tokens":42,"fertility":1.67,"p90":2.12,"cpt":3.1,"neglect":0.02}}}},{"code":"lug_Latn","name":"Ganda","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":4100000,"floor":41,"floors":{"flores":41},"metrics":{"flores":{"gpt2":{"tokens":54,"fertility":2.16,"p90":2.76,"cpt":2.3,"neglect":0.32},"cl100k":{"tokens":49,"fertility":1.96,"p90":2.47,"cpt":2.54,"neglect":0.2},"o200k":{"tokens":41,"fertility":1.65,"p90":2.08,"cpt":3.03,"neglect":0},"llama3":{"tokens":48,"fertility":1.94,"p90":2.42,"cpt":2.57,"neglect":0.17},"gemma":{"tokens":45,"fertility":1.81,"p90":2.29,"cpt":2.74,"neglect":0.1},"tekken":{"tokens":47,"fertility":1.86,"p90":2.35,"cpt":2.6,"neglect":0.15},"qwen2":{"tokens":49,"fertility":1.94,"p90":2.45,"cpt":2.52,"neglect":0.2},"bloom":{"tokens":42,"fertility":1.67,"p90":2.11,"cpt":2.95,"neglect":0.02}}}},{"code":"luo_Latn","name":"Luo (Kenya and Tanzania)","script":"Latin","family":"Nilotic","region":"Africa","speakers":3000000,"floor":42,"floors":{"flores":42},"metrics":{"flores":{"gpt2":{"tokens":51,"fertility":2,"p90":2.63,"cpt":2.5,"neglect":0.21},"cl100k":{"tokens":46,"fertility":1.82,"p90":2.33,"cpt":2.78,"neglect":0.1},"o200k":{"tokens":42,"fertility":1.65,"p90":2.16,"cpt":3.06,"neglect":0},"llama3":{"tokens":46,"fertility":1.81,"p90":2.33,"cpt":2.79,"neglect":0.1},"gemma":{"tokens":43,"fertility":1.68,"p90":2.18,"cpt":2.98,"neglect":0.02},"tekken":{"tokens":46,"fertility":1.75,"p90":2.26,"cpt":2.81,"neglect":0.1},"qwen2":{"tokens":46,"fertility":1.8,"p90":2.32,"cpt":2.76,"neglect":0.1},"bloom":{"tokens":42,"fertility":1.67,"p90":2.15,"cpt":3.01,"neglect":0}}}},{"code":"lus_Latn","name":"Mizo","script":"Latin","family":"Sino-Tibetan","region":"Eurasia","speakers":500000,"floor":44,"floors":{"flores":44},"metrics":{"flores":{"gpt2":{"tokens":51,"fertility":2.07,"p90":2.74,"cpt":2.57,"neglect":0.16},"cl100k":{"tokens":48,"fertility":1.95,"p90":2.55,"cpt":2.72,"neglect":0.09},"o200k":{"tokens":44,"fertility":1.79,"p90":2.33,"cpt":2.99,"neglect":0},"llama3":{"tokens":48,"fertility":1.9,"p90":2.5,"cpt":2.78,"neglect":0.09},"gemma":{"tokens":45,"fertility":1.79,"p90":2.37,"cpt":2.96,"neglect":0.02},"tekken":{"tokens":47,"fertility":1.86,"p90":2.44,"cpt":2.79,"neglect":0.07},"qwen2":{"tokens":48,"fertility":1.93,"p90":2.55,"cpt":2.7,"neglect":0.09},"bloom":{"tokens":46,"fertility":1.81,"p90":2.38,"cpt":2.9,"neglect":0.05}}}},{"code":"lvs_Latn","name":"Standard Latvian","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":1500000,"floor":44,"floors":{"flores":44,"massive":12},"metrics":{"flores":{"gpt2":{"tokens":63,"fertility":2.54,"p90":3.23,"cpt":1.95,"neglect":0.43},"cl100k":{"tokens":58,"fertility":2.36,"p90":3,"cpt":2.1,"neglect":0.32},"o200k":{"tokens":44,"fertility":1.81,"p90":2.27,"cpt":2.74,"neglect":0},"llama3":{"tokens":57,"fertility":2.31,"p90":2.94,"cpt":2.14,"neglect":0.3},"gemma":{"tokens":45,"fertility":1.81,"p90":2.28,"cpt":2.72,"neglect":0.02},"tekken":{"tokens":49,"fertility":1.91,"p90":2.41,"cpt":2.51,"neglect":0.11},"qwen2":{"tokens":58,"fertility":2.28,"p90":2.91,"cpt":2.13,"neglect":0.32},"bloom":{"tokens":52,"fertility":2.07,"p90":2.62,"cpt":2.36,"neglect":0.18}},"massive":{"gpt2":{"tokens":17,"fertility":2.43,"p90":3.67,"cpt":2,"neglect":0.42},"cl100k":{"tokens":15,"fertility":2.25,"p90":3.4,"cpt":2.19,"neglect":0.25},"o200k":{"tokens":12,"fertility":1.82,"p90":2.67,"cpt":2.77,"neglect":0},"llama3":{"tokens":15,"fertility":2.25,"p90":3.4,"cpt":2.21,"neglect":0.25},"gemma":{"tokens":12,"fertility":1.78,"p90":2.67,"cpt":2.82,"neglect":0},"tekken":{"tokens":13,"fertility":1.91,"p90":2.86,"cpt":2.53,"neglect":0.08},"qwen2":{"tokens":15,"fertility":2.22,"p90":3.33,"cpt":2.24,"neglect":0.25},"bloom":{"tokens":14,"fertility":2,"p90":3,"cpt":2.45,"neglect":0.17}}}},{"code":"mag_Deva","name":"Magahi","script":"Devanagari","family":"Indo-European","region":"Eurasia","speakers":20700000,"floor":37,"floors":{"flores":37},"metrics":{"flores":{"gpt2":{"tokens":179,"fertility":7.2,"p90":8.92,"cpt":0.65,"neglect":3.84},"cl100k":{"tokens":117,"fertility":4.69,"p90":5.79,"cpt":1,"neglect":2.16},"o200k":{"tokens":43,"fertility":1.7,"p90":2.08,"cpt":2.76,"neglect":0.16},"llama3":{"tokens":64,"fertility":2.56,"p90":3.13,"cpt":1.82,"neglect":0.73},"gemma":{"tokens":51,"fertility":2.04,"p90":2.52,"cpt":2.28,"neglect":0.38},"tekken":{"tokens":52,"fertility":2.04,"p90":2.48,"cpt":2.23,"neglect":0.41},"qwen2":{"tokens":107,"fertility":4.22,"p90":5.2,"cpt":1.09,"neglect":1.89},"bloom":{"tokens":37,"fertility":1.45,"p90":1.77,"cpt":3.18,"neglect":0}}}},{"code":"mai_Deva","name":"Maithili","script":"Devanagari","family":"Indo-European","region":"Eurasia","speakers":33900000,"floor":40,"floors":{"flores":40},"metrics":{"flores":{"gpt2":{"tokens":186,"fertility":7.38,"p90":9.36,"cpt":0.64,"neglect":3.65},"cl100k":{"tokens":123,"fertility":4.86,"p90":6.18,"cpt":0.97,"neglect":2.08},"o200k":{"tokens":44,"fertility":1.74,"p90":2.14,"cpt":2.74,"neglect":0.1},"llama3":{"tokens":65,"fertility":2.57,"p90":3.24,"cpt":1.84,"neglect":0.62},"gemma":{"tokens":57,"fertility":2.25,"p90":2.86,"cpt":2.1,"neglect":0.42},"tekken":{"tokens":58,"fertility":2.24,"p90":2.86,"cpt":2.06,"neglect":0.45},"qwen2":{"tokens":111,"fertility":4.36,"p90":5.57,"cpt":1.07,"neglect":1.77},"bloom":{"tokens":40,"fertility":1.56,"p90":1.94,"cpt":3.01,"neglect":0}}}},{"code":"mal_Mlym","name":"Malayalam","script":"Malayalam","family":"Dravidian","region":"Eurasia","speakers":37100000,"floor":34,"floors":{"flores":34,"massive":10},"metrics":{"flores":{"gpt2":{"tokens":376,"fertility":15.24,"p90":19.06,"cpt":0.36,"neglect":10.06},"cl100k":{"tokens":221,"fertility":9,"p90":11.25,"cpt":0.62,"neglect":5.5},"o200k":{"tokens":48,"fertility":1.94,"p90":2.41,"cpt":2.85,"neglect":0.41},"llama3":{"tokens":221,"fertility":9,"p90":11.25,"cpt":0.62,"neglect":5.5},"gemma":{"tokens":80,"fertility":3.21,"p90":4.05,"cpt":1.71,"neglect":1.35},"tekken":{"tokens":65,"fertility":2.55,"p90":3.21,"cpt":2.1,"neglect":0.91},"qwen2":{"tokens":182,"fertility":7.32,"p90":9.19,"cpt":0.75,"neglect":4.35},"bloom":{"tokens":34,"fertility":1.38,"p90":1.71,"cpt":3.95,"neglect":0}},"massive":{"gpt2":{"tokens":111,"fertility":16.11,"p90":23.17,"cpt":0.36,"neglect":10.1},"cl100k":{"tokens":65,"fertility":9.62,"p90":13.67,"cpt":0.61,"neglect":5.5},"o200k":{"tokens":14,"fertility":2.08,"p90":3,"cpt":2.87,"neglect":0.4},"llama3":{"tokens":64,"fertility":9.6,"p90":13.67,"cpt":0.62,"neglect":5.4},"gemma":{"tokens":23,"fertility":3.43,"p90":5,"cpt":1.74,"neglect":1.3},"tekken":{"tokens":19,"fertility":2.8,"p90":4,"cpt":2.05,"neglect":0.9},"qwen2":{"tokens":53,"fertility":8,"p90":11.25,"cpt":0.74,"neglect":4.3},"bloom":{"tokens":10,"fertility":1.44,"p90":2.2,"cpt":3.98,"neglect":0}}}},{"code":"mar_Deva","name":"Marathi","script":"Devanagari","family":"Indo-European","region":"Eurasia","speakers":83100000,"floor":30,"floors":{"flores":30},"metrics":{"flores":{"gpt2":{"tokens":192,"fertility":7.76,"p90":9.73,"cpt":0.63,"neglect":5.4},"cl100k":{"tokens":125,"fertility":5,"p90":6.29,"cpt":0.97,"neglect":3.17},"o200k":{"tokens":45,"fertility":1.8,"p90":2.23,"cpt":2.71,"neglect":0.5},"llama3":{"tokens":69,"fertility":2.77,"p90":3.43,"cpt":1.75,"neglect":1.3},"gemma":{"tokens":56,"fertility":2.24,"p90":2.82,"cpt":2.16,"neglect":0.87},"tekken":{"tokens":54,"fertility":2.14,"p90":2.65,"cpt":2.22,"neglect":0.8},"qwen2":{"tokens":116,"fertility":4.62,"p90":5.77,"cpt":1.04,"neglect":2.87},"bloom":{"tokens":30,"fertility":1.21,"p90":1.5,"cpt":3.97,"neglect":0}}}},{"code":"min_Arab","name":"Minangkabau","script":"Arabic","family":"Austronesian","region":"Papunesia","speakers":5530000,"floor":60,"floors":{"flores":60},"metrics":{"flores":{"gpt2":{"tokens":128,"fertility":5.18,"p90":6.32,"cpt":0.9,"neglect":1.13},"cl100k":{"tokens":98,"fertility":3.93,"p90":4.84,"cpt":1.18,"neglect":0.63},"o200k":{"tokens":65,"fertility":2.61,"p90":3.24,"cpt":1.78,"neglect":0.08},"llama3":{"tokens":72,"fertility":2.86,"p90":3.59,"cpt":1.61,"neglect":0.2},"gemma":{"tokens":60,"fertility":2.41,"p90":3.03,"cpt":1.92,"neglect":0},"tekken":{"tokens":65,"fertility":2.53,"p90":3.17,"cpt":1.78,"neglect":0.08},"qwen2":{"tokens":70,"fertility":2.73,"p90":3.41,"cpt":1.65,"neglect":0.17},"bloom":{"tokens":64,"fertility":2.53,"p90":3.16,"cpt":1.81,"neglect":0.07}}}},{"code":"min_Latn","name":"Minangkabau","script":"Latin","family":"Austronesian","region":"Papunesia","speakers":5530000,"floor":36,"floors":{"flores":36},"metrics":{"flores":{"gpt2":{"tokens":49,"fertility":1.95,"p90":2.43,"cpt":2.65,"neglect":0.36},"cl100k":{"tokens":44,"fertility":1.75,"p90":2.17,"cpt":2.94,"neglect":0.22},"o200k":{"tokens":38,"fertility":1.52,"p90":1.88,"cpt":3.4,"neglect":0.06},"llama3":{"tokens":44,"fertility":1.73,"p90":2.14,"cpt":2.97,"neglect":0.22},"gemma":{"tokens":37,"fertility":1.49,"p90":1.87,"cpt":3.44,"neglect":0.03},"tekken":{"tokens":41,"fertility":1.59,"p90":1.95,"cpt":3.17,"neglect":0.14},"qwen2":{"tokens":44,"fertility":1.74,"p90":2.16,"cpt":2.91,"neglect":0.22},"bloom":{"tokens":36,"fertility":1.43,"p90":1.76,"cpt":3.57,"neglect":0}}}},{"code":"mkd_Cyrl","name":"Macedonian","script":"Cyrillic","family":"Indo-European","region":"Eurasia","speakers":2000000,"floor":41,"floors":{"flores":41},"metrics":{"flores":{"gpt2":{"tokens":134,"fertility":5.41,"p90":6.79,"cpt":0.94,"neglect":2.27},"cl100k":{"tokens":69,"fertility":2.73,"p90":3.43,"cpt":1.84,"neglect":0.68},"o200k":{"tokens":43,"fertility":1.76,"p90":2.16,"cpt":2.88,"neglect":0.05},"llama3":{"tokens":52,"fertility":2.11,"p90":2.59,"cpt":2.39,"neglect":0.27},"gemma":{"tokens":44,"fertility":1.78,"p90":2.2,"cpt":2.84,"neglect":0.07},"tekken":{"tokens":41,"fertility":1.63,"p90":1.97,"cpt":3.02,"neglect":0},"qwen2":{"tokens":56,"fertility":2.24,"p90":2.8,"cpt":2.21,"neglect":0.37},"bloom":{"tokens":62,"fertility":2.48,"p90":3.08,"cpt":2.02,"neglect":0.51}}}},{"code":"mlt_Latn","name":"Maltese","script":"Latin","family":"Afro-Asiatic","region":"Eurasia","speakers":570000,"floor":46,"floors":{"flores":46},"metrics":{"flores":{"gpt2":{"tokens":66,"fertility":2.67,"p90":3.33,"cpt":2.01,"neglect":0.43},"cl100k":{"tokens":59,"fertility":2.38,"p90":3.01,"cpt":2.24,"neglect":0.28},"o200k":{"tokens":46,"fertility":1.84,"p90":2.25,"cpt":2.93,"neglect":0},"llama3":{"tokens":59,"fertility":2.36,"p90":3,"cpt":2.25,"neglect":0.28},"gemma":{"tokens":50,"fertility":2.03,"p90":2.5,"cpt":2.65,"neglect":0.09},"tekken":{"tokens":58,"fertility":2.29,"p90":2.91,"cpt":2.28,"neglect":0.26},"qwen2":{"tokens":56,"fertility":2.23,"p90":2.79,"cpt":2.37,"neglect":0.22},"bloom":{"tokens":56,"fertility":2.23,"p90":2.78,"cpt":2.38,"neglect":0.22}}}},{"code":"mni_Beng","name":"Manipuri","script":"Bengali","family":"Sino-Tibetan","region":"Eurasia","speakers":1470000,"floor":58,"floors":{"flores":58},"metrics":{"flores":{"gpt2":{"tokens":251,"fertility":9.96,"p90":12.96,"cpt":0.5,"neglect":3.33},"cl100k":{"tokens":167,"fertility":6.62,"p90":8.62,"cpt":0.75,"neglect":1.88},"o200k":{"tokens":76,"fertility":3.05,"p90":3.91,"cpt":1.66,"neglect":0.31},"llama3":{"tokens":166,"fertility":6.57,"p90":8.55,"cpt":0.75,"neglect":1.86},"gemma":{"tokens":91,"fertility":3.58,"p90":4.73,"cpt":1.38,"neglect":0.57},"tekken":{"tokens":86,"fertility":3.33,"p90":4.4,"cpt":1.45,"neglect":0.48},"qwen2":{"tokens":141,"fertility":5.56,"p90":7.22,"cpt":0.88,"neglect":1.43},"bloom":{"tokens":58,"fertility":2.33,"p90":3,"cpt":2.12,"neglect":0}}}},{"code":"mos_Latn","name":"Mossi","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":7600000,"floor":49,"floors":{"flores":49},"metrics":{"flores":{"gpt2":{"tokens":63,"fertility":2.52,"p90":3.48,"cpt":1.83,"neglect":0.29},"cl100k":{"tokens":58,"fertility":2.3,"p90":3.14,"cpt":2,"neglect":0.18},"o200k":{"tokens":51,"fertility":2.06,"p90":2.78,"cpt":2.27,"neglect":0.04},"llama3":{"tokens":54,"fertility":2.18,"p90":2.94,"cpt":2.12,"neglect":0.1},"gemma":{"tokens":49,"fertility":1.96,"p90":2.64,"cpt":2.35,"neglect":0},"tekken":{"tokens":54,"fertility":2.1,"p90":2.86,"cpt":2.14,"neglect":0.1},"qwen2":{"tokens":52,"fertility":2.08,"p90":2.77,"cpt":2.21,"neglect":0.06},"bloom":{"tokens":50,"fertility":2,"p90":2.69,"cpt":2.32,"neglect":0.02}}}},{"code":"mri_Latn","name":"Maori","script":"Latin","family":"Austronesian","region":"Papunesia","speakers":213849,"floor":45,"floors":{"flores":45},"metrics":{"flores":{"gpt2":{"tokens":60,"fertility":2.44,"p90":3.09,"cpt":2.22,"neglect":0.33},"cl100k":{"tokens":58,"fertility":2.35,"p90":2.96,"cpt":2.29,"neglect":0.29},"o200k":{"tokens":45,"fertility":1.8,"p90":2.24,"cpt":3,"neglect":0},"llama3":{"tokens":58,"fertility":2.33,"p90":2.95,"cpt":2.31,"neglect":0.29},"gemma":{"tokens":51,"fertility":2.04,"p90":2.59,"cpt":2.63,"neglect":0.13},"tekken":{"tokens":55,"fertility":2.16,"p90":2.75,"cpt":2.41,"neglect":0.22},"qwen2":{"tokens":59,"fertility":2.32,"p90":2.96,"cpt":2.27,"neglect":0.31},"bloom":{"tokens":53,"fertility":2.11,"p90":2.64,"cpt":2.53,"neglect":0.18}}}},{"code":"mya_Mymr","name":"Burmese","script":"Myanmar","family":"Sino-Tibetan","region":"Eurasia","speakers":32900000,"floor":78,"floors":{"flores":78,"massive":21},"metrics":{"flores":{"gpt2":{"tokens":414,"fertility":16.88,"p90":22.26,"cpt":0.36,"neglect":4.31},"cl100k":{"tokens":287,"fertility":11.7,"p90":15.22,"cpt":0.52,"neglect":2.68},"o200k":{"tokens":78,"fertility":3.19,"p90":4.08,"cpt":1.92,"neglect":0},"llama3":{"tokens":287,"fertility":11.7,"p90":15.22,"cpt":0.52,"neglect":2.68},"gemma":{"tokens":119,"fertility":4.83,"p90":6.31,"cpt":1.26,"neglect":0.53},"tekken":{"tokens":82,"fertility":3.27,"p90":4.18,"cpt":1.82,"neglect":0.05},"qwen2":{"tokens":224,"fertility":8.95,"p90":11.9,"cpt":0.66,"neglect":1.87},"bloom":{"tokens":249,"fertility":10.05,"p90":12.99,"cpt":0.6,"neglect":2.19}},"massive":{"gpt2":{"tokens":105,"fertility":15,"p90":22.8,"cpt":0.36,"neglect":4},"cl100k":{"tokens":72,"fertility":10.67,"p90":16.23,"cpt":0.53,"neglect":2.43},"o200k":{"tokens":21,"fertility":3.1,"p90":4.6,"cpt":1.82,"neglect":0},"llama3":{"tokens":72,"fertility":10.67,"p90":16.22,"cpt":0.53,"neglect":2.43},"gemma":{"tokens":30,"fertility":4.56,"p90":6.9,"cpt":1.24,"neglect":0.43},"tekken":{"tokens":24,"fertility":3.36,"p90":5,"cpt":1.61,"neglect":0.14},"qwen2":{"tokens":57,"fertility":8.33,"p90":12.88,"cpt":0.67,"neglect":1.71},"bloom":{"tokens":63,"fertility":9,"p90":13.71,"cpt":0.6,"neglect":2}}}},{"code":"nld_Latn","name":"Dutch","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":23100000,"floor":30,"floors":{"flores":30,"massive":8},"metrics":{"flores":{"gpt2":{"tokens":47,"fertility":1.94,"p90":2.46,"cpt":2.77,"neglect":0.57},"cl100k":{"tokens":39,"fertility":1.59,"p90":2,"cpt":3.39,"neglect":0.3},"o200k":{"tokens":30,"fertility":1.24,"p90":1.5,"cpt":4.4,"neglect":0},"llama3":{"tokens":39,"fertility":1.58,"p90":1.97,"cpt":3.41,"neglect":0.3},"gemma":{"tokens":32,"fertility":1.31,"p90":1.64,"cpt":4.09,"neglect":0.07},"tekken":{"tokens":35,"fertility":1.39,"p90":1.74,"cpt":3.74,"neglect":0.17},"qwen2":{"tokens":39,"fertility":1.57,"p90":1.96,"cpt":3.36,"neglect":0.3},"bloom":{"tokens":42,"fertility":1.7,"p90":2.13,"cpt":3.15,"neglect":0.4}},"massive":{"gpt2":{"tokens":13,"fertility":1.88,"p90":2.67,"cpt":2.75,"neglect":0.62},"cl100k":{"tokens":10,"fertility":1.5,"p90":2.2,"cpt":3.45,"neglect":0.25},"o200k":{"tokens":8,"fertility":1.2,"p90":1.71,"cpt":4.39,"neglect":0},"llama3":{"tokens":10,"fertility":1.5,"p90":2.2,"cpt":3.46,"neglect":0.25},"gemma":{"tokens":9,"fertility":1.25,"p90":1.83,"cpt":4.2,"neglect":0.12},"tekken":{"tokens":10,"fertility":1.43,"p90":2,"cpt":3.59,"neglect":0.25},"qwen2":{"tokens":10,"fertility":1.5,"p90":2.2,"cpt":3.46,"neglect":0.25},"bloom":{"tokens":11,"fertility":1.62,"p90":2.33,"cpt":3.17,"neglect":0.38}}}},{"code":"nno_Latn","name":"Norwegian Nynorsk","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":5300000,"floor":35,"floors":{"flores":35},"metrics":{"flores":{"gpt2":{"tokens":48,"fertility":1.92,"p90":2.38,"cpt":2.57,"neglect":0.37},"cl100k":{"tokens":41,"fertility":1.62,"p90":2,"cpt":3.02,"neglect":0.17},"o200k":{"tokens":35,"fertility":1.4,"p90":1.72,"cpt":3.55,"neglect":0},"llama3":{"tokens":41,"fertility":1.62,"p90":2,"cpt":3.04,"neglect":0.17},"gemma":{"tokens":35,"fertility":1.39,"p90":1.73,"cpt":3.51,"neglect":0},"tekken":{"tokens":38,"fertility":1.46,"p90":1.79,"cpt":3.28,"neglect":0.09},"qwen2":{"tokens":41,"fertility":1.62,"p90":2,"cpt":2.99,"neglect":0.17},"bloom":{"tokens":42,"fertility":1.65,"p90":2,"cpt":2.97,"neglect":0.2}}}},{"code":"nob_Latn","name":"Norwegian Bokmål","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":5300000,"floor":33,"floors":{"flores":33,"massive":9},"metrics":{"flores":{"gpt2":{"tokens":46,"fertility":1.85,"p90":2.3,"cpt":2.66,"neglect":0.39},"cl100k":{"tokens":39,"fertility":1.56,"p90":1.91,"cpt":3.14,"neglect":0.18},"o200k":{"tokens":33,"fertility":1.35,"p90":1.65,"cpt":3.67,"neglect":0},"llama3":{"tokens":39,"fertility":1.55,"p90":1.91,"cpt":3.16,"neglect":0.18},"gemma":{"tokens":33,"fertility":1.33,"p90":1.64,"cpt":3.66,"neglect":0},"tekken":{"tokens":36,"fertility":1.41,"p90":1.73,"cpt":3.37,"neglect":0.09},"qwen2":{"tokens":40,"fertility":1.55,"p90":1.91,"cpt":3.11,"neglect":0.21},"bloom":{"tokens":40,"fertility":1.61,"p90":2,"cpt":3.02,"neglect":0.21}},"massive":{"gpt2":{"tokens":12,"fertility":1.7,"p90":2.4,"cpt":2.67,"neglect":0.33},"cl100k":{"tokens":10,"fertility":1.46,"p90":2,"cpt":3.21,"neglect":0.11},"o200k":{"tokens":9,"fertility":1.27,"p90":1.75,"cpt":3.7,"neglect":0},"llama3":{"tokens":10,"fertility":1.44,"p90":2,"cpt":3.22,"neglect":0.11},"gemma":{"tokens":9,"fertility":1.25,"p90":1.75,"cpt":3.78,"neglect":0},"tekken":{"tokens":10,"fertility":1.33,"p90":1.89,"cpt":3.35,"neglect":0.11},"qwen2":{"tokens":10,"fertility":1.46,"p90":2,"cpt":3.21,"neglect":0.11},"bloom":{"tokens":11,"fertility":1.5,"p90":2,"cpt":3.01,"neglect":0.22}}}},{"code":"npi_Deva","name":"Nepali","script":"Devanagari","family":"Indo-European","region":"Eurasia","speakers":15800000,"floor":29,"floors":{"flores":29},"metrics":{"flores":{"gpt2":{"tokens":190,"fertility":7.58,"p90":9.64,"cpt":0.62,"neglect":5.55},"cl100k":{"tokens":120,"fertility":4.75,"p90":6.01,"cpt":0.98,"neglect":3.14},"o200k":{"tokens":40,"fertility":1.61,"p90":2,"cpt":2.93,"neglect":0.38},"llama3":{"tokens":65,"fertility":2.58,"p90":3.24,"cpt":1.81,"neglect":1.24},"gemma":{"tokens":53,"fertility":2.12,"p90":2.67,"cpt":2.2,"neglect":0.83},"tekken":{"tokens":55,"fertility":2.14,"p90":2.7,"cpt":2.13,"neglect":0.9},"qwen2":{"tokens":113,"fertility":4.41,"p90":5.65,"cpt":1.04,"neglect":2.9},"bloom":{"tokens":29,"fertility":1.18,"p90":1.45,"cpt":3.92,"neglect":0}}}},{"code":"nso_Latn","name":"Pedi","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":4100000,"floor":45,"floors":{"flores":45},"metrics":{"flores":{"gpt2":{"tokens":57,"fertility":2.3,"p90":2.97,"cpt":2.42,"neglect":0.27},"cl100k":{"tokens":54,"fertility":2.19,"p90":2.79,"cpt":2.56,"neglect":0.2},"o200k":{"tokens":45,"fertility":1.85,"p90":2.31,"cpt":3.04,"neglect":0},"llama3":{"tokens":53,"fertility":2.16,"p90":2.73,"cpt":2.6,"neglect":0.18},"gemma":{"tokens":48,"fertility":1.98,"p90":2.5,"cpt":2.82,"neglect":0.07},"tekken":{"tokens":51,"fertility":2.06,"p90":2.6,"cpt":2.65,"neglect":0.13},"qwen2":{"tokens":54,"fertility":2.17,"p90":2.79,"cpt":2.54,"neglect":0.2},"bloom":{"tokens":48,"fertility":1.92,"p90":2.42,"cpt":2.86,"neglect":0.07}}}},{"code":"nus_Latn","name":"Nuer","script":"Latin","family":"Nilotic","region":"Africa","speakers":900000,"floor":68,"floors":{"flores":68},"metrics":{"flores":{"gpt2":{"tokens":103,"fertility":4.16,"p90":5.56,"cpt":1.26,"neglect":0.51},"cl100k":{"tokens":98,"fertility":3.94,"p90":5.3,"cpt":1.32,"neglect":0.44},"o200k":{"tokens":78,"fertility":3.18,"p90":4.33,"cpt":1.64,"neglect":0.15},"llama3":{"tokens":97,"fertility":3.93,"p90":5.29,"cpt":1.33,"neglect":0.43},"gemma":{"tokens":71,"fertility":2.87,"p90":3.83,"cpt":1.81,"neglect":0.04},"tekken":{"tokens":95,"fertility":3.74,"p90":5,"cpt":1.36,"neglect":0.4},"qwen2":{"tokens":84,"fertility":3.35,"p90":4.52,"cpt":1.53,"neglect":0.24},"bloom":{"tokens":68,"fertility":2.76,"p90":3.62,"cpt":1.87,"neglect":0}}}},{"code":"nya_Latn","name":"Nyanja","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":12000000,"floor":40,"floors":{"flores":40},"metrics":{"flores":{"gpt2":{"tokens":56,"fertility":2.24,"p90":2.92,"cpt":2.43,"neglect":0.4},"cl100k":{"tokens":52,"fertility":2.08,"p90":2.69,"cpt":2.63,"neglect":0.3},"o200k":{"tokens":40,"fertility":1.62,"p90":2.07,"cpt":3.41,"neglect":0},"llama3":{"tokens":52,"fertility":2.05,"p90":2.67,"cpt":2.66,"neglect":0.3},"gemma":{"tokens":45,"fertility":1.83,"p90":2.35,"cpt":2.99,"neglect":0.12},"tekken":{"tokens":51,"fertility":1.97,"p90":2.56,"cpt":2.7,"neglect":0.28},"qwen2":{"tokens":53,"fertility":2.06,"p90":2.67,"cpt":2.61,"neglect":0.33},"bloom":{"tokens":45,"fertility":1.77,"p90":2.26,"cpt":3.05,"neglect":0.12}}}},{"code":"oci_Latn","name":"Occitan","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":542000,"floor":37,"floors":{"flores":37},"metrics":{"flores":{"gpt2":{"tokens":51,"fertility":2.05,"p90":2.56,"cpt":2.69,"neglect":0.38},"cl100k":{"tokens":45,"fertility":1.82,"p90":2.22,"cpt":3.03,"neglect":0.22},"o200k":{"tokens":39,"fertility":1.58,"p90":1.94,"cpt":3.5,"neglect":0.05},"llama3":{"tokens":45,"fertility":1.81,"p90":2.21,"cpt":3.04,"neglect":0.22},"gemma":{"tokens":41,"fertility":1.65,"p90":2.04,"cpt":3.32,"neglect":0.11},"tekken":{"tokens":42,"fertility":1.62,"p90":2,"cpt":3.29,"neglect":0.14},"qwen2":{"tokens":45,"fertility":1.8,"p90":2.21,"cpt":3.01,"neglect":0.22},"bloom":{"tokens":37,"fertility":1.48,"p90":1.77,"cpt":3.69,"neglect":0}}}},{"code":"ory_Orya","name":"Odia","script":"Odia","family":"Indo-European","region":"Eurasia","speakers":34500000,"floor":34,"floors":{"flores":34},"metrics":{"flores":{"gpt2":{"tokens":329,"fertility":13.31,"p90":17,"cpt":0.38,"neglect":8.68},"cl100k":{"tokens":309,"fertility":12.43,"p90":16,"cpt":0.4,"neglect":8.09},"o200k":{"tokens":123,"fertility":4.98,"p90":6.32,"cpt":1.01,"neglect":2.62},"llama3":{"tokens":302,"fertility":12.18,"p90":15.73,"cpt":0.41,"neglect":7.88},"gemma":{"tokens":123,"fertility":5,"p90":6.39,"cpt":1,"neglect":2.62},"tekken":{"tokens":329,"fertility":13,"p90":16.68,"cpt":0.38,"neglect":8.68},"qwen2":{"tokens":245,"fertility":9.75,"p90":12.63,"cpt":0.5,"neglect":6.21},"bloom":{"tokens":34,"fertility":1.36,"p90":1.67,"cpt":3.62,"neglect":0}}}},{"code":"pag_Latn","name":"Pangasinan","script":"Latin","family":"Austronesian","region":"Papunesia","speakers":1100000,"floor":36,"floors":{"flores":36},"metrics":{"flores":{"gpt2":{"tokens":41,"fertility":1.66,"p90":2.25,"cpt":2.93,"neglect":0.14},"cl100k":{"tokens":39,"fertility":1.57,"p90":2.11,"cpt":3.09,"neglect":0.08},"o200k":{"tokens":36,"fertility":1.44,"p90":1.91,"cpt":3.4,"neglect":0},"llama3":{"tokens":39,"fertility":1.57,"p90":2.08,"cpt":3.12,"neglect":0.08},"gemma":{"tokens":36,"fertility":1.45,"p90":1.91,"cpt":3.34,"neglect":0},"tekken":{"tokens":38,"fertility":1.5,"p90":2,"cpt":3.16,"neglect":0.06},"qwen2":{"tokens":39,"fertility":1.56,"p90":2.1,"cpt":3.06,"neglect":0.08},"bloom":{"tokens":37,"fertility":1.45,"p90":1.92,"cpt":3.32,"neglect":0.03}}}},{"code":"pan_Guru","name":"Eastern Panjabi","script":"Gurmukhi","family":"Indo-European","region":"Eurasia","speakers":125000000,"floor":36,"floors":{"flores":36},"metrics":{"flores":{"gpt2":{"tokens":194,"fertility":7.83,"p90":9.78,"cpt":0.63,"neglect":4.39},"cl100k":{"tokens":194,"fertility":7.86,"p90":9.78,"cpt":0.63,"neglect":4.39},"o200k":{"tokens":64,"fertility":2.6,"p90":3.2,"cpt":1.89,"neglect":0.78},"llama3":{"tokens":193,"fertility":7.8,"p90":9.73,"cpt":0.63,"neglect":4.36},"gemma":{"tokens":81,"fertility":3.22,"p90":4.03,"cpt":1.52,"neglect":1.25},"tekken":{"tokens":76,"fertility":3,"p90":3.64,"cpt":1.61,"neglect":1.11},"qwen2":{"tokens":182,"fertility":7.33,"p90":9.2,"cpt":0.66,"neglect":4.06},"bloom":{"tokens":36,"fertility":1.43,"p90":1.75,"cpt":3.37,"neglect":0}}}},{"code":"pap_Latn","name":"Papiamento","script":"Latin","family":"Indo-European","region":"South America","speakers":321300,"floor":39,"floors":{"flores":39},"metrics":{"flores":{"gpt2":{"tokens":50,"fertility":1.97,"p90":2.43,"cpt":2.6,"neglect":0.28},"cl100k":{"tokens":43,"fertility":1.75,"p90":2.13,"cpt":2.93,"neglect":0.1},"o200k":{"tokens":39,"fertility":1.58,"p90":1.9,"cpt":3.27,"neglect":0},"llama3":{"tokens":43,"fertility":1.73,"p90":2.12,"cpt":2.95,"neglect":0.1},"gemma":{"tokens":39,"fertility":1.57,"p90":1.93,"cpt":3.25,"neglect":0},"tekken":{"tokens":41,"fertility":1.61,"p90":1.95,"cpt":3.1,"neglect":0.05},"qwen2":{"tokens":44,"fertility":1.73,"p90":2.12,"cpt":2.91,"neglect":0.13},"bloom":{"tokens":39,"fertility":1.54,"p90":1.86,"cpt":3.29,"neglect":0}}}},{"code":"pbt_Arab","name":"Southern Pashto","script":"Arabic","family":"Indo-European","region":"Eurasia","speakers":10900000,"floor":43,"floors":{"flores":43},"metrics":{"flores":{"gpt2":{"tokens":134,"fertility":5.45,"p90":6.72,"cpt":0.87,"neglect":2.12},"cl100k":{"tokens":95,"fertility":3.87,"p90":4.75,"cpt":1.22,"neglect":1.21},"o200k":{"tokens":43,"fertility":1.75,"p90":2.15,"cpt":2.7,"neglect":0},"llama3":{"tokens":67,"fertility":2.71,"p90":3.41,"cpt":1.74,"neglect":0.56},"gemma":{"tokens":55,"fertility":2.23,"p90":2.79,"cpt":2.11,"neglect":0.28},"tekken":{"tokens":51,"fertility":2,"p90":2.5,"cpt":2.28,"neglect":0.19},"qwen2":{"tokens":73,"fertility":2.91,"p90":3.61,"cpt":1.6,"neglect":0.7},"bloom":{"tokens":64,"fertility":2.56,"p90":3.23,"cpt":1.83,"neglect":0.49}}}},{"code":"pes_Arab","name":"Western Persian","script":"Arabic","family":"Indo-European","region":"Eurasia","speakers":57000000,"floor":36,"floors":{"flores":36,"massive":10},"metrics":{"flores":{"gpt2":{"tokens":132,"fertility":5.31,"p90":6.63,"cpt":0.87,"neglect":2.67},"cl100k":{"tokens":82,"fertility":3.27,"p90":4.15,"cpt":1.4,"neglect":1.28},"o200k":{"tokens":38,"fertility":1.52,"p90":1.91,"cpt":3.03,"neglect":0.06},"llama3":{"tokens":38,"fertility":1.52,"p90":1.9,"cpt":3.02,"neglect":0.06},"gemma":{"tokens":36,"fertility":1.44,"p90":1.82,"cpt":3.17,"neglect":0},"tekken":{"tokens":37,"fertility":1.44,"p90":1.77,"cpt":3.1,"neglect":0.03},"qwen2":{"tokens":66,"fertility":2.59,"p90":3.29,"cpt":1.75,"neglect":0.83},"bloom":{"tokens":45,"fertility":1.78,"p90":2.26,"cpt":2.57,"neglect":0.25}},"massive":{"gpt2":{"tokens":36,"fertility":5.29,"p90":7.67,"cpt":0.85,"neglect":2.6},"cl100k":{"tokens":22,"fertility":3.33,"p90":4.71,"cpt":1.39,"neglect":1.2},"o200k":{"tokens":11,"fertility":1.6,"p90":2.38,"cpt":2.9,"neglect":0.1},"llama3":{"tokens":11,"fertility":1.57,"p90":2.33,"cpt":2.92,"neglect":0.1},"gemma":{"tokens":10,"fertility":1.44,"p90":2.17,"cpt":3.22,"neglect":0},"tekken":{"tokens":11,"fertility":1.57,"p90":2.25,"cpt":2.89,"neglect":0.1},"qwen2":{"tokens":18,"fertility":2.67,"p90":3.8,"cpt":1.74,"neglect":0.8},"bloom":{"tokens":12,"fertility":1.71,"p90":2.5,"cpt":2.59,"neglect":0.2}}}},{"code":"plt_Latn","name":"Plateau Malagasy","script":"Latin","family":"Austronesian","region":"Africa","speakers":18000000,"floor":44,"floors":{"flores":44},"metrics":{"flores":{"gpt2":{"tokens":64,"fertility":2.58,"p90":3.31,"cpt":2.32,"neglect":0.45},"cl100k":{"tokens":56,"fertility":2.25,"p90":2.86,"cpt":2.64,"neglect":0.27},"o200k":{"tokens":44,"fertility":1.78,"p90":2.27,"cpt":3.36,"neglect":0},"llama3":{"tokens":55,"fertility":2.22,"p90":2.84,"cpt":2.67,"neglect":0.25},"gemma":{"tokens":50,"fertility":2,"p90":2.61,"cpt":2.94,"neglect":0.14},"tekken":{"tokens":53,"fertility":2.12,"p90":2.7,"cpt":2.75,"neglect":0.2},"qwen2":{"tokens":56,"fertility":2.21,"p90":2.85,"cpt":2.63,"neglect":0.27},"bloom":{"tokens":52,"fertility":2.07,"p90":2.61,"cpt":2.86,"neglect":0.18}}}},{"code":"pol_Latn","name":"Polish","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":39700000,"floor":36,"floors":{"flores":36,"massive":9},"metrics":{"flores":{"gpt2":{"tokens":67,"fertility":2.69,"p90":3.48,"cpt":1.93,"neglect":0.86},"cl100k":{"tokens":48,"fertility":1.92,"p90":2.41,"cpt":2.7,"neglect":0.33},"o200k":{"tokens":41,"fertility":1.64,"p90":2.05,"cpt":3.19,"neglect":0.14},"llama3":{"tokens":48,"fertility":1.89,"p90":2.38,"cpt":2.74,"neglect":0.33},"gemma":{"tokens":36,"fertility":1.47,"p90":1.84,"cpt":3.54,"neglect":0},"tekken":{"tokens":41,"fertility":1.58,"p90":2,"cpt":3.19,"neglect":0.14},"qwen2":{"tokens":45,"fertility":1.77,"p90":2.25,"cpt":2.88,"neglect":0.25},"bloom":{"tokens":54,"fertility":2.13,"p90":2.69,"cpt":2.4,"neglect":0.5}},"massive":{"gpt2":{"tokens":17,"fertility":2.46,"p90":3.67,"cpt":1.96,"neglect":0.89},"cl100k":{"tokens":12,"fertility":1.8,"p90":2.67,"cpt":2.71,"neglect":0.33},"o200k":{"tokens":10,"fertility":1.56,"p90":2.25,"cpt":3.18,"neglect":0.11},"llama3":{"tokens":12,"fertility":1.8,"p90":2.6,"cpt":2.73,"neglect":0.33},"gemma":{"tokens":9,"fertility":1.38,"p90":2,"cpt":3.58,"neglect":0},"tekken":{"tokens":11,"fertility":1.57,"p90":2.25,"cpt":3.05,"neglect":0.22},"qwen2":{"tokens":11,"fertility":1.7,"p90":2.5,"cpt":2.89,"neglect":0.22},"bloom":{"tokens":13,"fertility":2,"p90":2.83,"cpt":2.47,"neglect":0.44}}}},{"code":"por_Latn","name":"Portuguese","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":254300000,"floor":28,"floors":{"flores":28,"massive":8},"metrics":{"flores":{"gpt2":{"tokens":48,"fertility":1.92,"p90":2.38,"cpt":2.74,"neglect":0.71},"cl100k":{"tokens":36,"fertility":1.47,"p90":1.79,"cpt":3.56,"neglect":0.29},"o200k":{"tokens":30,"fertility":1.22,"p90":1.47,"cpt":4.35,"neglect":0.07},"llama3":{"tokens":36,"fertility":1.46,"p90":1.79,"cpt":3.57,"neglect":0.29},"gemma":{"tokens":30,"fertility":1.21,"p90":1.48,"cpt":4.31,"neglect":0.07},"tekken":{"tokens":32,"fertility":1.26,"p90":1.53,"cpt":4.05,"neglect":0.14},"qwen2":{"tokens":37,"fertility":1.44,"p90":1.77,"cpt":3.57,"neglect":0.32},"bloom":{"tokens":28,"fertility":1.11,"p90":1.33,"cpt":4.66,"neglect":0}},"massive":{"gpt2":{"tokens":13,"fertility":1.86,"p90":2.67,"cpt":2.72,"neglect":0.62},"cl100k":{"tokens":10,"fertility":1.43,"p90":2,"cpt":3.6,"neglect":0.25},"o200k":{"tokens":8,"fertility":1.22,"p90":1.71,"cpt":4.32,"neglect":0},"llama3":{"tokens":10,"fertility":1.43,"p90":2,"cpt":3.61,"neglect":0.25},"gemma":{"tokens":8,"fertility":1.18,"p90":1.67,"cpt":4.44,"neglect":0},"tekken":{"tokens":9,"fertility":1.29,"p90":1.8,"cpt":3.95,"neglect":0.12},"qwen2":{"tokens":10,"fertility":1.42,"p90":2,"cpt":3.66,"neglect":0.25},"bloom":{"tokens":8,"fertility":1.14,"p90":1.6,"cpt":4.45,"neglect":0}}}},{"code":"prs_Arab","name":"Dari","script":"Arabic","family":"Indo-European","region":"Eurasia","speakers":9600000,"floor":35,"floors":{"flores":35},"metrics":{"flores":{"gpt2":{"tokens":127,"fertility":5.06,"p90":6.36,"cpt":0.88,"neglect":2.63},"cl100k":{"tokens":79,"fertility":3.14,"p90":3.96,"cpt":1.42,"neglect":1.26},"o200k":{"tokens":36,"fertility":1.46,"p90":1.8,"cpt":3.07,"neglect":0.03},"llama3":{"tokens":37,"fertility":1.46,"p90":1.82,"cpt":3.01,"neglect":0.06},"gemma":{"tokens":35,"fertility":1.41,"p90":1.76,"cpt":3.14,"neglect":0},"tekken":{"tokens":36,"fertility":1.4,"p90":1.75,"cpt":3.07,"neglect":0.03},"qwen2":{"tokens":63,"fertility":2.48,"p90":3.13,"cpt":1.76,"neglect":0.8},"bloom":{"tokens":41,"fertility":1.64,"p90":2.04,"cpt":2.7,"neglect":0.17}}}},{"code":"quy_Latn","name":"Ayacucho Quechua","script":"Latin","family":"Quechuan","region":"South America","speakers":918200,"floor":45,"floors":{"flores":45},"metrics":{"flores":{"gpt2":{"tokens":55,"fertility":2.17,"p90":2.86,"cpt":2.37,"neglect":0.22},"cl100k":{"tokens":52,"fertility":2.07,"p90":2.73,"cpt":2.5,"neglect":0.16},"o200k":{"tokens":45,"fertility":1.82,"p90":2.37,"cpt":2.87,"neglect":0},"llama3":{"tokens":51,"fertility":2.05,"p90":2.7,"cpt":2.52,"neglect":0.13},"gemma":{"tokens":45,"fertility":1.8,"p90":2.35,"cpt":2.85,"neglect":0},"tekken":{"tokens":49,"fertility":1.93,"p90":2.56,"cpt":2.61,"neglect":0.09},"qwen2":{"tokens":52,"fertility":2.04,"p90":2.71,"cpt":2.48,"neglect":0.16},"bloom":{"tokens":46,"fertility":1.83,"p90":2.4,"cpt":2.81,"neglect":0.02}}}},{"code":"ron_Latn","name":"Romanian","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":24300000,"floor":39,"floors":{"flores":39,"massive":9},"metrics":{"flores":{"gpt2":{"tokens":61,"fertility":2.44,"p90":3.17,"cpt":2.22,"neglect":0.56},"cl100k":{"tokens":46,"fertility":1.87,"p90":2.31,"cpt":2.92,"neglect":0.18},"o200k":{"tokens":40,"fertility":1.62,"p90":2,"cpt":3.4,"neglect":0.03},"llama3":{"tokens":46,"fertility":1.86,"p90":2.29,"cpt":2.93,"neglect":0.18},"gemma":{"tokens":39,"fertility":1.54,"p90":1.88,"cpt":3.53,"neglect":0},"tekken":{"tokens":41,"fertility":1.6,"p90":1.97,"cpt":3.33,"neglect":0.05},"qwen2":{"tokens":47,"fertility":1.86,"p90":2.31,"cpt":2.89,"neglect":0.21},"bloom":{"tokens":47,"fertility":1.89,"p90":2.37,"cpt":2.85,"neglect":0.21}},"massive":{"gpt2":{"tokens":13,"fertility":1.86,"p90":3,"cpt":2.53,"neglect":0.44},"cl100k":{"tokens":11,"fertility":1.6,"p90":2.33,"cpt":3.1,"neglect":0.22},"o200k":{"tokens":9,"fertility":1.43,"p90":2,"cpt":3.55,"neglect":0},"llama3":{"tokens":11,"fertility":1.6,"p90":2.33,"cpt":3.12,"neglect":0.22},"gemma":{"tokens":9,"fertility":1.4,"p90":2,"cpt":3.64,"neglect":0},"tekken":{"tokens":10,"fertility":1.45,"p90":2,"cpt":3.33,"neglect":0.11},"qwen2":{"tokens":11,"fertility":1.6,"p90":2.33,"cpt":3.1,"neglect":0.22},"bloom":{"tokens":11,"fertility":1.55,"p90":2.29,"cpt":3.11,"neglect":0.22}}}},{"code":"run_Latn","name":"Rundi","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":10800000,"floor":41,"floors":{"flores":41},"metrics":{"flores":{"gpt2":{"tokens":57,"fertility":2.31,"p90":2.92,"cpt":2.35,"neglect":0.39},"cl100k":{"tokens":53,"fertility":2.12,"p90":2.68,"cpt":2.54,"neglect":0.29},"o200k":{"tokens":43,"fertility":1.76,"p90":2.24,"cpt":3.09,"neglect":0.05},"llama3":{"tokens":52,"fertility":2.11,"p90":2.67,"cpt":2.56,"neglect":0.27},"gemma":{"tokens":48,"fertility":1.96,"p90":2.5,"cpt":2.75,"neglect":0.17},"tekken":{"tokens":51,"fertility":2,"p90":2.56,"cpt":2.62,"neglect":0.24},"qwen2":{"tokens":53,"fertility":2.11,"p90":2.67,"cpt":2.53,"neglect":0.29},"bloom":{"tokens":41,"fertility":1.62,"p90":2.07,"cpt":3.28,"neglect":0}}}},{"code":"rus_Cyrl","name":"Russian","script":"Cyrillic","family":"Indo-European","region":"Eurasia","speakers":154000000,"floor":34,"floors":{"flores":34,"massive":8},"metrics":{"flores":{"gpt2":{"tokens":144,"fertility":5.74,"p90":7.43,"cpt":0.93,"neglect":3.24},"cl100k":{"tokens":62,"fertility":2.47,"p90":3.14,"cpt":2.14,"neglect":0.82},"o200k":{"tokens":35,"fertility":1.41,"p90":1.79,"cpt":3.77,"neglect":0.03},"llama3":{"tokens":41,"fertility":1.62,"p90":2.05,"cpt":3.26,"neglect":0.21},"gemma":{"tokens":34,"fertility":1.38,"p90":1.75,"cpt":3.83,"neglect":0},"tekken":{"tokens":39,"fertility":1.51,"p90":1.9,"cpt":3.42,"neglect":0.15},"qwen2":{"tokens":44,"fertility":1.76,"p90":2.21,"cpt":2.97,"neglect":0.29},"bloom":{"tokens":63,"fertility":2.48,"p90":3.12,"cpt":2.13,"neglect":0.85}},"massive":{"gpt2":{"tokens":36,"fertility":5.25,"p90":7.56,"cpt":0.92,"neglect":3.5},"cl100k":{"tokens":16,"fertility":2.33,"p90":3.43,"cpt":2.12,"neglect":1},"o200k":{"tokens":9,"fertility":1.33,"p90":2,"cpt":3.69,"neglect":0.12},"llama3":{"tokens":11,"fertility":1.6,"p90":2.5,"cpt":3.09,"neglect":0.38},"gemma":{"tokens":8,"fertility":1.27,"p90":1.91,"cpt":3.96,"neglect":0},"tekken":{"tokens":10,"fertility":1.5,"p90":2.2,"cpt":3.3,"neglect":0.25},"qwen2":{"tokens":11,"fertility":1.67,"p90":2.5,"cpt":2.94,"neglect":0.38},"bloom":{"tokens":16,"fertility":2.33,"p90":3.38,"cpt":2.09,"neglect":1}}}},{"code":"sag_Latn","name":"Sango","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":4600000,"floor":44,"floors":{"flores":44},"metrics":{"flores":{"gpt2":{"tokens":54,"fertility":2.17,"p90":2.93,"cpt":2.39,"neglect":0.23},"cl100k":{"tokens":51,"fertility":2.05,"p90":2.73,"cpt":2.54,"neglect":0.16},"o200k":{"tokens":44,"fertility":1.77,"p90":2.38,"cpt":2.95,"neglect":0},"llama3":{"tokens":51,"fertility":2.03,"p90":2.7,"cpt":2.57,"neglect":0.16},"gemma":{"tokens":45,"fertility":1.79,"p90":2.37,"cpt":2.91,"neglect":0.02},"tekken":{"tokens":49,"fertility":1.92,"p90":2.53,"cpt":2.65,"neglect":0.11},"qwen2":{"tokens":51,"fertility":2,"p90":2.68,"cpt":2.55,"neglect":0.16},"bloom":{"tokens":44,"fertility":1.76,"p90":2.35,"cpt":2.93,"neglect":0}}}},{"code":"san_Deva","name":"Sanskrit","script":"Devanagari","family":"Indo-European","region":"Eurasia","speakers":49736,"floor":41,"floors":{"flores":41},"metrics":{"flores":{"gpt2":{"tokens":196,"fertility":7.91,"p90":10.03,"cpt":0.6,"neglect":3.78},"cl100k":{"tokens":124,"fertility":5,"p90":6.27,"cpt":0.96,"neglect":2.02},"o200k":{"tokens":51,"fertility":2.07,"p90":2.61,"cpt":2.32,"neglect":0.24},"llama3":{"tokens":66,"fertility":2.66,"p90":3.32,"cpt":1.8,"neglect":0.61},"gemma":{"tokens":56,"fertility":2.3,"p90":2.86,"cpt":2.1,"neglect":0.37},"tekken":{"tokens":58,"fertility":2.29,"p90":2.88,"cpt":2.04,"neglect":0.41},"qwen2":{"tokens":116,"fertility":4.58,"p90":5.77,"cpt":1.03,"neglect":1.83},"bloom":{"tokens":41,"fertility":1.62,"p90":2.04,"cpt":2.91,"neglect":0}}}},{"code":"sat_Olck","name":"Santali","script":"Ol Chiki","family":"Austroasiatic","region":"Eurasia","speakers":7200000,"floor":128,"floors":{"flores":128},"metrics":{"flores":{"gpt2":{"tokens":320,"fertility":12.86,"p90":15.92,"cpt":0.4,"neglect":1.5},"cl100k":{"tokens":320,"fertility":12.84,"p90":16,"cpt":0.4,"neglect":1.5},"o200k":{"tokens":339,"fertility":13.74,"p90":17.24,"cpt":0.38,"neglect":1.65},"llama3":{"tokens":320,"fertility":12.84,"p90":16,"cpt":0.4,"neglect":1.5},"gemma":{"tokens":128,"fertility":5.19,"p90":6.54,"cpt":1,"neglect":0},"tekken":{"tokens":320,"fertility":12.54,"p90":15.83,"cpt":0.4,"neglect":1.5},"qwen2":{"tokens":226,"fertility":9,"p90":11.26,"cpt":0.57,"neglect":0.77},"bloom":{"tokens":319,"fertility":12.75,"p90":15.72,"cpt":0.4,"neglect":1.49}}}},{"code":"scn_Latn","name":"Sicilian","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":4700000,"floor":44,"floors":{"flores":44},"metrics":{"flores":{"gpt2":{"tokens":56,"fertility":2.24,"p90":2.86,"cpt":2.28,"neglect":0.27},"cl100k":{"tokens":50,"fertility":1.97,"p90":2.53,"cpt":2.56,"neglect":0.14},"o200k":{"tokens":45,"fertility":1.8,"p90":2.27,"cpt":2.86,"neglect":0.02},"llama3":{"tokens":49,"fertility":1.96,"p90":2.51,"cpt":2.59,"neglect":0.11},"gemma":{"tokens":44,"fertility":1.77,"p90":2.24,"cpt":2.89,"neglect":0},"tekken":{"tokens":47,"fertility":1.83,"p90":2.33,"cpt":2.71,"neglect":0.07},"qwen2":{"tokens":49,"fertility":1.93,"p90":2.45,"cpt":2.59,"neglect":0.11},"bloom":{"tokens":46,"fertility":1.79,"p90":2.24,"cpt":2.83,"neglect":0.05}}}},{"code":"shn_Mymr","name":"Shan","script":"Myanmar","family":"Tai-Kadai","region":"Eurasia","speakers":3000000,"floor":148,"floors":{"flores":148},"metrics":{"flores":{"gpt2":{"tokens":468,"fertility":18.84,"p90":25.02,"cpt":0.37,"neglect":2.16},"cl100k":{"tokens":377,"fertility":15.11,"p90":20.28,"cpt":0.46,"neglect":1.55},"o200k":{"tokens":197,"fertility":7.96,"p90":10.69,"cpt":0.88,"neglect":0.33},"llama3":{"tokens":371,"fertility":14.94,"p90":20,"cpt":0.47,"neglect":1.51},"gemma":{"tokens":148,"fertility":5.93,"p90":7.93,"cpt":1.17,"neglect":0},"tekken":{"tokens":236,"fertility":9.32,"p90":12.54,"cpt":0.73,"neglect":0.59},"qwen2":{"tokens":265,"fertility":10.59,"p90":14.09,"cpt":0.65,"neglect":0.79},"bloom":{"tokens":304,"fertility":12.14,"p90":15.97,"cpt":0.57,"neglect":1.05}}}},{"code":"sin_Sinh","name":"Sinhala","script":"Sinhala","family":"Indo-European","region":"Eurasia","speakers":15300000,"floor":68,"floors":{"flores":68},"metrics":{"flores":{"gpt2":{"tokens":318,"fertility":12.86,"p90":16.01,"cpt":0.38,"neglect":3.68},"cl100k":{"tokens":219,"fertility":8.88,"p90":11.06,"cpt":0.56,"neglect":2.22},"o200k":{"tokens":68,"fertility":2.74,"p90":3.37,"cpt":1.81,"neglect":0},"llama3":{"tokens":218,"fertility":8.83,"p90":11,"cpt":0.56,"neglect":2.21},"gemma":{"tokens":102,"fertility":4.15,"p90":5.14,"cpt":1.19,"neglect":0.5},"tekken":{"tokens":319,"fertility":12.64,"p90":15.68,"cpt":0.38,"neglect":3.69},"qwen2":{"tokens":176,"fertility":7.05,"p90":8.76,"cpt":0.69,"neglect":1.59},"bloom":{"tokens":205,"fertility":8.28,"p90":10.23,"cpt":0.59,"neglect":2.01}}}},{"code":"slk_Latn","name":"Slovak","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":6000000,"floor":39,"floors":{"flores":39},"metrics":{"flores":{"gpt2":{"tokens":62,"fertility":2.5,"p90":3.26,"cpt":1.95,"neglect":0.59},"cl100k":{"tokens":53,"fertility":2.13,"p90":2.71,"cpt":2.29,"neglect":0.36},"o200k":{"tokens":41,"fertility":1.68,"p90":2.1,"cpt":2.94,"neglect":0.05},"llama3":{"tokens":47,"fertility":1.85,"p90":2.35,"cpt":2.64,"neglect":0.21},"gemma":{"tokens":39,"fertility":1.58,"p90":1.98,"cpt":3.08,"neglect":0},"tekken":{"tokens":43,"fertility":1.68,"p90":2.11,"cpt":2.84,"neglect":0.1},"qwen2":{"tokens":53,"fertility":2.09,"p90":2.67,"cpt":2.31,"neglect":0.36},"bloom":{"tokens":50,"fertility":2,"p90":2.55,"cpt":2.41,"neglect":0.28}}}},{"code":"slv_Latn","name":"Slovenian","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":2400000,"floor":37,"floors":{"flores":37,"massive":10},"metrics":{"flores":{"gpt2":{"tokens":52,"fertility":2.11,"p90":2.68,"cpt":2.31,"neglect":0.41},"cl100k":{"tokens":47,"fertility":1.87,"p90":2.39,"cpt":2.57,"neglect":0.27},"o200k":{"tokens":37,"fertility":1.5,"p90":1.88,"cpt":3.25,"neglect":0},"llama3":{"tokens":45,"fertility":1.78,"p90":2.28,"cpt":2.7,"neglect":0.22},"gemma":{"tokens":39,"fertility":1.56,"p90":1.96,"cpt":3.09,"neglect":0.05},"tekken":{"tokens":40,"fertility":1.55,"p90":1.95,"cpt":3.03,"neglect":0.08},"qwen2":{"tokens":47,"fertility":1.85,"p90":2.38,"cpt":2.56,"neglect":0.27},"bloom":{"tokens":46,"fertility":1.79,"p90":2.29,"cpt":2.66,"neglect":0.24}},"massive":{"gpt2":{"tokens":14,"fertility":2,"p90":3,"cpt":2.27,"neglect":0.4},"cl100k":{"tokens":13,"fertility":1.83,"p90":2.75,"cpt":2.58,"neglect":0.3},"o200k":{"tokens":10,"fertility":1.5,"p90":2.2,"cpt":3.25,"neglect":0},"llama3":{"tokens":12,"fertility":1.75,"p90":2.67,"cpt":2.7,"neglect":0.2},"gemma":{"tokens":10,"fertility":1.5,"p90":2.25,"cpt":3.17,"neglect":0},"tekken":{"tokens":11,"fertility":1.55,"p90":2.29,"cpt":3.01,"neglect":0.1},"qwen2":{"tokens":13,"fertility":1.83,"p90":2.75,"cpt":2.59,"neglect":0.3},"bloom":{"tokens":12,"fertility":1.71,"p90":2.5,"cpt":2.72,"neglect":0.2}}}},{"code":"smo_Latn","name":"Samoan","script":"Latin","family":"Austronesian","region":"Papunesia","speakers":415720,"floor":47,"floors":{"flores":47},"metrics":{"flores":{"gpt2":{"tokens":63,"fertility":2.57,"p90":3.3,"cpt":2.22,"neglect":0.34},"cl100k":{"tokens":56,"fertility":2.29,"p90":2.9,"cpt":2.48,"neglect":0.19},"o200k":{"tokens":47,"fertility":1.91,"p90":2.41,"cpt":2.99,"neglect":0},"llama3":{"tokens":55,"fertility":2.26,"p90":2.87,"cpt":2.52,"neglect":0.17},"gemma":{"tokens":52,"fertility":2.11,"p90":2.71,"cpt":2.68,"neglect":0.11},"tekken":{"tokens":54,"fertility":2.15,"p90":2.74,"cpt":2.56,"neglect":0.15},"qwen2":{"tokens":57,"fertility":2.27,"p90":2.89,"cpt":2.47,"neglect":0.21},"bloom":{"tokens":53,"fertility":2.12,"p90":2.67,"cpt":2.65,"neglect":0.13}}}},{"code":"sna_Latn","name":"Shona","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":8300000,"floor":42,"floors":{"flores":42},"metrics":{"flores":{"gpt2":{"tokens":56,"fertility":2.28,"p90":2.98,"cpt":2.39,"neglect":0.33},"cl100k":{"tokens":52,"fertility":2.12,"p90":2.79,"cpt":2.56,"neglect":0.24},"o200k":{"tokens":42,"fertility":1.68,"p90":2.14,"cpt":3.24,"neglect":0},"llama3":{"tokens":51,"fertility":2.06,"p90":2.7,"cpt":2.62,"neglect":0.21},"gemma":{"tokens":46,"fertility":1.86,"p90":2.41,"cpt":2.92,"neglect":0.1},"tekken":{"tokens":50,"fertility":1.97,"p90":2.56,"cpt":2.69,"neglect":0.19},"qwen2":{"tokens":53,"fertility":2.1,"p90":2.77,"cpt":2.54,"neglect":0.26},"bloom":{"tokens":44,"fertility":1.79,"p90":2.31,"cpt":3,"neglect":0.05}}}},{"code":"snd_Arab","name":"Sindhi","script":"Arabic","family":"Indo-European","region":"Eurasia","speakers":24600000,"floor":42,"floors":{"flores":42},"metrics":{"flores":{"gpt2":{"tokens":123,"fertility":4.96,"p90":6.12,"cpt":0.89,"neglect":1.93},"cl100k":{"tokens":99,"fertility":4,"p90":5,"cpt":1.1,"neglect":1.36},"o200k":{"tokens":42,"fertility":1.7,"p90":2.05,"cpt":2.6,"neglect":0},"llama3":{"tokens":72,"fertility":2.83,"p90":3.6,"cpt":1.54,"neglect":0.71},"gemma":{"tokens":59,"fertility":2.36,"p90":2.96,"cpt":1.86,"neglect":0.4},"tekken":{"tokens":64,"fertility":2.5,"p90":3.17,"cpt":1.71,"neglect":0.52},"qwen2":{"tokens":71,"fertility":2.85,"p90":3.56,"cpt":1.52,"neglect":0.69},"bloom":{"tokens":63,"fertility":2.48,"p90":3.17,"cpt":1.74,"neglect":0.5}}}},{"code":"som_Latn","name":"Somali","script":"Latin","family":"Afro-Asiatic","region":"Africa","speakers":16200000,"floor":42,"floors":{"flores":42},"metrics":{"flores":{"gpt2":{"tokens":58,"fertility":2.33,"p90":2.96,"cpt":2.37,"neglect":0.38},"cl100k":{"tokens":54,"fertility":2.15,"p90":2.73,"cpt":2.55,"neglect":0.29},"o200k":{"tokens":42,"fertility":1.68,"p90":2.08,"cpt":3.29,"neglect":0},"llama3":{"tokens":53,"fertility":2.14,"p90":2.69,"cpt":2.57,"neglect":0.26},"gemma":{"tokens":47,"fertility":1.9,"p90":2.41,"cpt":2.9,"neglect":0.12},"tekken":{"tokens":54,"fertility":2.1,"p90":2.67,"cpt":2.55,"neglect":0.29},"qwen2":{"tokens":54,"fertility":2.14,"p90":2.72,"cpt":2.53,"neglect":0.29},"bloom":{"tokens":50,"fertility":2,"p90":2.53,"cpt":2.72,"neglect":0.19}}}},{"code":"sot_Latn","name":"Southern Sotho","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":6000000,"floor":45,"floors":{"flores":45},"metrics":{"flores":{"gpt2":{"tokens":58,"fertility":2.34,"p90":2.94,"cpt":2.52,"neglect":0.29},"cl100k":{"tokens":55,"fertility":2.22,"p90":2.78,"cpt":2.65,"neglect":0.22},"o200k":{"tokens":45,"fertility":1.83,"p90":2.3,"cpt":3.22,"neglect":0},"llama3":{"tokens":55,"fertility":2.2,"p90":2.75,"cpt":2.68,"neglect":0.22},"gemma":{"tokens":49,"fertility":1.97,"p90":2.51,"cpt":2.96,"neglect":0.09},"tekken":{"tokens":54,"fertility":2.12,"p90":2.7,"cpt":2.69,"neglect":0.2},"qwen2":{"tokens":55,"fertility":2.2,"p90":2.77,"cpt":2.63,"neglect":0.22},"bloom":{"tokens":49,"fertility":1.95,"p90":2.43,"cpt":2.97,"neglect":0.09}}}},{"code":"spa_Latn","name":"Spanish","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":485000000,"floor":31,"floors":{"flores":31,"massive":8},"metrics":{"flores":{"gpt2":{"tokens":50,"fertility":1.98,"p90":2.5,"cpt":2.92,"neglect":0.61},"cl100k":{"tokens":38,"fertility":1.54,"p90":1.9,"cpt":3.74,"neglect":0.23},"o200k":{"tokens":32,"fertility":1.3,"p90":1.62,"cpt":4.45,"neglect":0.03},"llama3":{"tokens":38,"fertility":1.54,"p90":1.9,"cpt":3.75,"neglect":0.23},"gemma":{"tokens":31,"fertility":1.25,"p90":1.56,"cpt":4.58,"neglect":0},"tekken":{"tokens":33,"fertility":1.3,"p90":1.6,"cpt":4.29,"neglect":0.06},"qwen2":{"tokens":39,"fertility":1.52,"p90":1.88,"cpt":3.73,"neglect":0.26},"bloom":{"tokens":31,"fertility":1.2,"p90":1.47,"cpt":4.74,"neglect":0}},"massive":{"gpt2":{"tokens":13,"fertility":1.8,"p90":2.6,"cpt":2.84,"neglect":0.62},"cl100k":{"tokens":9,"fertility":1.38,"p90":2,"cpt":3.87,"neglect":0.12},"o200k":{"tokens":8,"fertility":1.2,"p90":1.67,"cpt":4.45,"neglect":0},"llama3":{"tokens":9,"fertility":1.38,"p90":2,"cpt":3.87,"neglect":0.12},"gemma":{"tokens":8,"fertility":1.14,"p90":1.6,"cpt":4.74,"neglect":0},"tekken":{"tokens":9,"fertility":1.25,"p90":1.75,"cpt":4.17,"neglect":0.12},"qwen2":{"tokens":9,"fertility":1.38,"p90":2,"cpt":3.88,"neglect":0.12},"bloom":{"tokens":8,"fertility":1.17,"p90":1.6,"cpt":4.53,"neglect":0}}}},{"code":"srd_Latn","name":"Sardinian","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":1000000,"floor":43,"floors":{"flores":43},"metrics":{"flores":{"gpt2":{"tokens":56,"fertility":2.27,"p90":2.81,"cpt":2.51,"neglect":0.3},"cl100k":{"tokens":49,"fertility":2,"p90":2.46,"cpt":2.84,"neglect":0.14},"o200k":{"tokens":44,"fertility":1.78,"p90":2.2,"cpt":3.2,"neglect":0.02},"llama3":{"tokens":49,"fertility":1.96,"p90":2.43,"cpt":2.88,"neglect":0.14},"gemma":{"tokens":44,"fertility":1.78,"p90":2.17,"cpt":3.2,"neglect":0.02},"tekken":{"tokens":46,"fertility":1.81,"p90":2.22,"cpt":3.04,"neglect":0.07},"qwen2":{"tokens":49,"fertility":2,"p90":2.44,"cpt":2.82,"neglect":0.14},"bloom":{"tokens":43,"fertility":1.74,"p90":2.12,"cpt":3.23,"neglect":0}}}},{"code":"srp_Cyrl","name":"Serbian","script":"Cyrillic","family":"Indo-European","region":"Eurasia","speakers":9000000,"floor":41,"floors":{"flores":41},"metrics":{"flores":{"gpt2":{"tokens":131,"fertility":5.3,"p90":6.6,"cpt":0.91,"neglect":2.2},"cl100k":{"tokens":71,"fertility":2.91,"p90":3.62,"cpt":1.66,"neglect":0.73},"o200k":{"tokens":43,"fertility":1.76,"p90":2.19,"cpt":2.74,"neglect":0.05},"llama3":{"tokens":54,"fertility":2.18,"p90":2.72,"cpt":2.2,"neglect":0.32},"gemma":{"tokens":43,"fertility":1.75,"p90":2.12,"cpt":2.76,"neglect":0.05},"tekken":{"tokens":41,"fertility":1.64,"p90":2,"cpt":2.87,"neglect":0},"qwen2":{"tokens":59,"fertility":2.35,"p90":2.91,"cpt":2.03,"neglect":0.44},"bloom":{"tokens":64,"fertility":2.55,"p90":3.17,"cpt":1.87,"neglect":0.56}}}},{"code":"ssw_Latn","name":"Swati","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":2034200,"floor":44,"floors":{"flores":44},"metrics":{"flores":{"gpt2":{"tokens":57,"fertility":2.27,"p90":2.93,"cpt":2.37,"neglect":0.3},"cl100k":{"tokens":53,"fertility":2.12,"p90":2.75,"cpt":2.53,"neglect":0.2},"o200k":{"tokens":44,"fertility":1.77,"p90":2.26,"cpt":3.08,"neglect":0},"llama3":{"tokens":53,"fertility":2.11,"p90":2.72,"cpt":2.56,"neglect":0.2},"gemma":{"tokens":48,"fertility":1.91,"p90":2.46,"cpt":2.81,"neglect":0.09},"tekken":{"tokens":52,"fertility":2,"p90":2.56,"cpt":2.63,"neglect":0.18},"qwen2":{"tokens":54,"fertility":2.1,"p90":2.73,"cpt":2.52,"neglect":0.23},"bloom":{"tokens":45,"fertility":1.8,"p90":2.29,"cpt":2.98,"neglect":0.02}}}},{"code":"sun_Latn","name":"Sundanese","script":"Latin","family":"Austronesian","region":"Papunesia","speakers":32400000,"floor":34,"floors":{"flores":34},"metrics":{"flores":{"gpt2":{"tokens":51,"fertility":2.02,"p90":2.52,"cpt":2.52,"neglect":0.5},"cl100k":{"tokens":46,"fertility":1.82,"p90":2.27,"cpt":2.78,"neglect":0.35},"o200k":{"tokens":34,"fertility":1.38,"p90":1.7,"cpt":3.72,"neglect":0},"llama3":{"tokens":45,"fertility":1.81,"p90":2.25,"cpt":2.81,"neglect":0.32},"gemma":{"tokens":38,"fertility":1.5,"p90":1.89,"cpt":3.35,"neglect":0.12},"tekken":{"tokens":43,"fertility":1.69,"p90":2.12,"cpt":2.94,"neglect":0.26},"qwen2":{"tokens":46,"fertility":1.81,"p90":2.25,"cpt":2.76,"neglect":0.35},"bloom":{"tokens":37,"fertility":1.48,"p90":1.87,"cpt":3.39,"neglect":0.09}}}},{"code":"swe_Latn","name":"Swedish","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":9244250,"floor":32,"floors":{"flores":32,"massive":8},"metrics":{"flores":{"gpt2":{"tokens":48,"fertility":1.96,"p90":2.46,"cpt":2.51,"neglect":0.5},"cl100k":{"tokens":39,"fertility":1.58,"p90":1.94,"cpt":3.1,"neglect":0.22},"o200k":{"tokens":33,"fertility":1.35,"p90":1.64,"cpt":3.66,"neglect":0.03},"llama3":{"tokens":39,"fertility":1.57,"p90":1.93,"cpt":3.12,"neglect":0.22},"gemma":{"tokens":32,"fertility":1.31,"p90":1.6,"cpt":3.74,"neglect":0},"tekken":{"tokens":36,"fertility":1.41,"p90":1.72,"cpt":3.39,"neglect":0.12},"qwen2":{"tokens":39,"fertility":1.57,"p90":1.93,"cpt":3.07,"neglect":0.22},"bloom":{"tokens":42,"fertility":1.65,"p90":2.04,"cpt":2.93,"neglect":0.31}},"massive":{"gpt2":{"tokens":13,"fertility":1.86,"p90":2.57,"cpt":2.46,"neglect":0.62},"cl100k":{"tokens":10,"fertility":1.5,"p90":2,"cpt":3.07,"neglect":0.25},"o200k":{"tokens":9,"fertility":1.31,"p90":1.8,"cpt":3.64,"neglect":0.12},"llama3":{"tokens":10,"fertility":1.5,"p90":2,"cpt":3.1,"neglect":0.25},"gemma":{"tokens":8,"fertility":1.25,"p90":1.75,"cpt":3.79,"neglect":0},"tekken":{"tokens":10,"fertility":1.38,"p90":1.89,"cpt":3.31,"neglect":0.25},"qwen2":{"tokens":10,"fertility":1.5,"p90":2,"cpt":3.08,"neglect":0.25},"bloom":{"tokens":11,"fertility":1.57,"p90":2.17,"cpt":2.9,"neglect":0.38}}}},{"code":"swh_Latn","name":"Swahili","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":18000000,"floor":31,"floors":{"flores":31,"massive":9},"metrics":{"flores":{"gpt2":{"tokens":53,"fertility":2.13,"p90":2.61,"cpt":2.4,"neglect":0.71},"cl100k":{"tokens":48,"fertility":1.96,"p90":2.39,"cpt":2.62,"neglect":0.55},"o200k":{"tokens":36,"fertility":1.47,"p90":1.78,"cpt":3.48,"neglect":0.16},"llama3":{"tokens":48,"fertility":1.94,"p90":2.36,"cpt":2.64,"neglect":0.55},"gemma":{"tokens":40,"fertility":1.59,"p90":1.96,"cpt":3.19,"neglect":0.29},"tekken":{"tokens":46,"fertility":1.82,"p90":2.22,"cpt":2.74,"neglect":0.48},"qwen2":{"tokens":49,"fertility":1.94,"p90":2.38,"cpt":2.6,"neglect":0.58},"bloom":{"tokens":31,"fertility":1.23,"p90":1.5,"cpt":4.07,"neglect":0}},"massive":{"gpt2":{"tokens":14,"fertility":2,"p90":3,"cpt":2.43,"neglect":0.56},"cl100k":{"tokens":13,"fertility":2,"p90":2.83,"cpt":2.64,"neglect":0.44},"o200k":{"tokens":10,"fertility":1.5,"p90":2.17,"cpt":3.53,"neglect":0.11},"llama3":{"tokens":13,"fertility":1.93,"p90":2.8,"cpt":2.67,"neglect":0.44},"gemma":{"tokens":11,"fertility":1.6,"p90":2.33,"cpt":3.26,"neglect":0.22},"tekken":{"tokens":13,"fertility":1.8,"p90":2.6,"cpt":2.77,"neglect":0.44},"qwen2":{"tokens":13,"fertility":2,"p90":2.83,"cpt":2.64,"neglect":0.44},"bloom":{"tokens":9,"fertility":1.25,"p90":1.83,"cpt":3.98,"neglect":0}}}},{"code":"szl_Latn","name":"Silesian","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":457900,"floor":47,"floors":{"flores":47},"metrics":{"flores":{"gpt2":{"tokens":65,"fertility":2.57,"p90":3.35,"cpt":1.96,"neglect":0.38},"cl100k":{"tokens":55,"fertility":2.17,"p90":2.79,"cpt":2.32,"neglect":0.17},"o200k":{"tokens":48,"fertility":1.93,"p90":2.48,"cpt":2.61,"neglect":0.02},"llama3":{"tokens":53,"fertility":2.11,"p90":2.72,"cpt":2.38,"neglect":0.13},"gemma":{"tokens":47,"fertility":1.86,"p90":2.4,"cpt":2.69,"neglect":0},"tekken":{"tokens":50,"fertility":1.93,"p90":2.5,"cpt":2.54,"neglect":0.06},"qwen2":{"tokens":53,"fertility":2.08,"p90":2.68,"cpt":2.38,"neglect":0.13},"bloom":{"tokens":54,"fertility":2.14,"p90":2.73,"cpt":2.33,"neglect":0.15}}}},{"code":"tam_Taml","name":"Tamil","script":"Tamil","family":"Dravidian","region":"Eurasia","speakers":75000000,"floor":32,"floors":{"flores":32,"massive":9},"metrics":{"flores":{"gpt2":{"tokens":386,"fertility":15.53,"p90":19.37,"cpt":0.37,"neglect":11.06},"cl100k":{"tokens":192,"fertility":7.61,"p90":9.58,"cpt":0.74,"neglect":5},"o200k":{"tokens":48,"fertility":1.96,"p90":2.43,"cpt":2.9,"neglect":0.5},"llama3":{"tokens":192,"fertility":7.61,"p90":9.6,"cpt":0.74,"neglect":5},"gemma":{"tokens":65,"fertility":2.57,"p90":3.22,"cpt":2.19,"neglect":1.03},"tekken":{"tokens":56,"fertility":2.17,"p90":2.68,"cpt":2.54,"neglect":0.75},"qwen2":{"tokens":156,"fertility":6.15,"p90":7.74,"cpt":0.91,"neglect":3.88},"bloom":{"tokens":32,"fertility":1.28,"p90":1.59,"cpt":4.36,"neglect":0}},"massive":{"gpt2":{"tokens":108,"fertility":16,"p90":22.67,"cpt":0.36,"neglect":11},"cl100k":{"tokens":53,"fertility":8,"p90":11.5,"cpt":0.73,"neglect":4.89},"o200k":{"tokens":13,"fertility":2,"p90":3,"cpt":2.88,"neglect":0.44},"llama3":{"tokens":53,"fertility":8,"p90":11.5,"cpt":0.73,"neglect":4.89},"gemma":{"tokens":17,"fertility":2.67,"p90":4,"cpt":2.22,"neglect":0.89},"tekken":{"tokens":16,"fertility":2.36,"p90":3.46,"cpt":2.39,"neglect":0.78},"qwen2":{"tokens":43,"fertility":6.5,"p90":9.2,"cpt":0.9,"neglect":3.78},"bloom":{"tokens":9,"fertility":1.33,"p90":2,"cpt":4.27,"neglect":0}}}},{"code":"taq_Latn","name":"Tamasheq","script":"Latin","family":"Afro-Asiatic","region":"Africa","speakers":500000,"floor":45,"floors":{"flores":45},"metrics":{"flores":{"gpt2":{"tokens":56,"fertility":2.31,"p90":3.32,"cpt":1.95,"neglect":0.24},"cl100k":{"tokens":53,"fertility":2.16,"p90":3.06,"cpt":2.08,"neglect":0.18},"o200k":{"tokens":50,"fertility":2.03,"p90":2.89,"cpt":2.25,"neglect":0.11},"llama3":{"tokens":53,"fertility":2.13,"p90":3.01,"cpt":2.12,"neglect":0.18},"gemma":{"tokens":45,"fertility":1.86,"p90":2.56,"cpt":2.45,"neglect":0},"tekken":{"tokens":52,"fertility":2.06,"p90":2.88,"cpt":2.15,"neglect":0.16},"qwen2":{"tokens":49,"fertility":1.96,"p90":2.68,"cpt":2.29,"neglect":0.09},"bloom":{"tokens":46,"fertility":1.89,"p90":2.56,"cpt":2.42,"neglect":0.02}}}},{"code":"taq_Tfng","name":"Tamasheq","script":"Tifinagh","family":"Afro-Asiatic","region":"Africa","speakers":500000,"floor":132,"floors":{"flores":132},"metrics":{"flores":{"gpt2":{"tokens":255,"fertility":10.33,"p90":13.69,"cpt":0.44,"neglect":0.93},"cl100k":{"tokens":248,"fertility":10.04,"p90":13.26,"cpt":0.45,"neglect":0.88},"o200k":{"tokens":248,"fertility":10.23,"p90":13.41,"cpt":0.45,"neglect":0.88},"llama3":{"tokens":248,"fertility":10.05,"p90":13.26,"cpt":0.45,"neglect":0.88},"gemma":{"tokens":132,"fertility":5.36,"p90":7.16,"cpt":0.86,"neglect":0},"tekken":{"tokens":246,"fertility":9.8,"p90":12.85,"cpt":0.46,"neglect":0.86},"qwen2":{"tokens":132,"fertility":5.32,"p90":7,"cpt":0.85,"neglect":0},"bloom":{"tokens":190,"fertility":7.67,"p90":10.05,"cpt":0.59,"neglect":0.44}}}},{"code":"tat_Cyrl","name":"Tatar","script":"Cyrillic","family":"Turkic","region":"Eurasia","speakers":5200000,"floor":43,"floors":{"flores":43},"metrics":{"flores":{"gpt2":{"tokens":144,"fertility":5.76,"p90":7.37,"cpt":0.85,"neglect":2.35},"cl100k":{"tokens":92,"fertility":3.72,"p90":4.86,"cpt":1.32,"neglect":1.14},"o200k":{"tokens":43,"fertility":1.74,"p90":2.17,"cpt":2.85,"neglect":0},"llama3":{"tokens":78,"fertility":3.08,"p90":4.11,"cpt":1.59,"neglect":0.81},"gemma":{"tokens":55,"fertility":2.22,"p90":2.84,"cpt":2.22,"neglect":0.28},"tekken":{"tokens":54,"fertility":2.12,"p90":2.69,"cpt":2.26,"neglect":0.26},"qwen2":{"tokens":72,"fertility":2.87,"p90":3.71,"cpt":1.69,"neglect":0.67},"bloom":{"tokens":78,"fertility":3.12,"p90":3.98,"cpt":1.56,"neglect":0.81}}}},{"code":"tel_Telu","name":"Telugu","script":"Telugu","family":"Dravidian","region":"Eurasia","speakers":82000000,"floor":34,"floors":{"flores":34,"massive":10},"metrics":{"flores":{"gpt2":{"tokens":325,"fertility":13.08,"p90":16.47,"cpt":0.38,"neglect":8.56},"cl100k":{"tokens":208,"fertility":8.33,"p90":10.63,"cpt":0.59,"neglect":5.12},"o200k":{"tokens":48,"fertility":1.92,"p90":2.44,"cpt":2.56,"neglect":0.41},"llama3":{"tokens":208,"fertility":8.33,"p90":10.63,"cpt":0.59,"neglect":5.12},"gemma":{"tokens":73,"fertility":2.89,"p90":3.7,"cpt":1.7,"neglect":1.15},"tekken":{"tokens":58,"fertility":2.28,"p90":2.9,"cpt":2.1,"neglect":0.71},"qwen2":{"tokens":179,"fertility":7.06,"p90":9.04,"cpt":0.69,"neglect":4.26},"bloom":{"tokens":34,"fertility":1.34,"p90":1.68,"cpt":3.61,"neglect":0}},"massive":{"gpt2":{"tokens":96,"fertility":13.92,"p90":20,"cpt":0.37,"neglect":8.6},"cl100k":{"tokens":61,"fertility":9,"p90":13,"cpt":0.58,"neglect":5.1},"o200k":{"tokens":14,"fertility":2.12,"p90":3.23,"cpt":2.47,"neglect":0.4},"llama3":{"tokens":61,"fertility":9,"p90":13,"cpt":0.58,"neglect":5.1},"gemma":{"tokens":21,"fertility":3.17,"p90":4.67,"cpt":1.66,"neglect":1.1},"tekken":{"tokens":18,"fertility":2.62,"p90":3.86,"cpt":1.94,"neglect":0.8},"qwen2":{"tokens":53,"fertility":7.83,"p90":11.33,"cpt":0.66,"neglect":4.3},"bloom":{"tokens":10,"fertility":1.47,"p90":2.33,"cpt":3.44,"neglect":0}}}},{"code":"tgk_Cyrl","name":"Tajik","script":"Cyrillic","family":"Indo-European","region":"Eurasia","speakers":14000000,"floor":47,"floors":{"flores":47},"metrics":{"flores":{"gpt2":{"tokens":152,"fertility":6.06,"p90":7.77,"cpt":0.89,"neglect":2.23},"cl100k":{"tokens":91,"fertility":3.62,"p90":4.64,"cpt":1.49,"neglect":0.94},"o200k":{"tokens":47,"fertility":1.88,"p90":2.36,"cpt":2.89,"neglect":0},"llama3":{"tokens":73,"fertility":2.93,"p90":3.8,"cpt":1.84,"neglect":0.55},"gemma":{"tokens":62,"fertility":2.47,"p90":3.18,"cpt":2.17,"neglect":0.32},"tekken":{"tokens":69,"fertility":2.68,"p90":3.44,"cpt":1.96,"neglect":0.47},"qwen2":{"tokens":74,"fertility":2.91,"p90":3.71,"cpt":1.83,"neglect":0.57},"bloom":{"tokens":83,"fertility":3.25,"p90":4.21,"cpt":1.63,"neglect":0.77}}}},{"code":"tgl_Latn","name":"Tagalog","script":"Latin","family":"Austronesian","region":"Papunesia","speakers":23600000,"floor":41,"floors":{"flores":41,"massive":11},"metrics":{"flores":{"gpt2":{"tokens":56,"fertility":2.24,"p90":2.83,"cpt":2.72,"neglect":0.37},"cl100k":{"tokens":51,"fertility":2.05,"p90":2.57,"cpt":2.98,"neglect":0.24},"o200k":{"tokens":41,"fertility":1.67,"p90":2.08,"cpt":3.68,"neglect":0},"llama3":{"tokens":50,"fertility":2.03,"p90":2.54,"cpt":3.02,"neglect":0.22},"gemma":{"tokens":44,"fertility":1.76,"p90":2.22,"cpt":3.45,"neglect":0.07},"tekken":{"tokens":49,"fertility":1.9,"p90":2.39,"cpt":3.12,"neglect":0.2},"qwen2":{"tokens":51,"fertility":2.04,"p90":2.57,"cpt":2.96,"neglect":0.24},"bloom":{"tokens":46,"fertility":1.85,"p90":2.28,"cpt":3.28,"neglect":0.12}},"massive":{"gpt2":{"tokens":14,"fertility":2,"p90":3,"cpt":2.83,"neglect":0.27},"cl100k":{"tokens":13,"fertility":2,"p90":2.83,"cpt":3.11,"neglect":0.18},"o200k":{"tokens":11,"fertility":1.6,"p90":2.33,"cpt":3.77,"neglect":0},"llama3":{"tokens":13,"fertility":1.9,"p90":2.8,"cpt":3.16,"neglect":0.18},"gemma":{"tokens":11,"fertility":1.67,"p90":2.5,"cpt":3.64,"neglect":0},"tekken":{"tokens":13,"fertility":1.8,"p90":2.67,"cpt":3.24,"neglect":0.18},"qwen2":{"tokens":13,"fertility":2,"p90":2.83,"cpt":3.11,"neglect":0.18},"bloom":{"tokens":12,"fertility":1.75,"p90":2.5,"cpt":3.37,"neglect":0.09}}}},{"code":"tha_Thai","name":"Thai","script":"Thai","family":"Tai-Kadai","region":"Eurasia","speakers":20700000,"floor":44,"floors":{"flores":44,"massive":11},"metrics":{"flores":{"gpt2":{"tokens":221,"fertility":8.91,"p90":11.52,"cpt":0.52,"neglect":4.02},"cl100k":{"tokens":107,"fertility":4.31,"p90":5.58,"cpt":1.08,"neglect":1.43},"o200k":{"tokens":47,"fertility":1.92,"p90":2.47,"cpt":2.43,"neglect":0.07},"llama3":{"tokens":55,"fertility":2.18,"p90":2.75,"cpt":2.12,"neglect":0.25},"gemma":{"tokens":44,"fertility":1.79,"p90":2.35,"cpt":2.56,"neglect":0},"tekken":{"tokens":59,"fertility":2.34,"p90":2.96,"cpt":1.93,"neglect":0.34},"qwen2":{"tokens":64,"fertility":2.53,"p90":3.26,"cpt":1.79,"neglect":0.45},"bloom":{"tokens":114,"fertility":4.57,"p90":5.84,"cpt":1.01,"neglect":1.59}},"massive":{"gpt2":{"tokens":55,"fertility":8,"p90":11.75,"cpt":0.53,"neglect":4},"cl100k":{"tokens":27,"fertility":4,"p90":6,"cpt":1.07,"neglect":1.45},"o200k":{"tokens":12,"fertility":1.75,"p90":2.67,"cpt":2.47,"neglect":0.09},"llama3":{"tokens":14,"fertility":2.08,"p90":3,"cpt":2.08,"neglect":0.27},"gemma":{"tokens":11,"fertility":1.69,"p90":2.6,"cpt":2.57,"neglect":0},"tekken":{"tokens":16,"fertility":2.27,"p90":3.33,"cpt":1.86,"neglect":0.45},"qwen2":{"tokens":17,"fertility":2.5,"p90":3.75,"cpt":1.73,"neglect":0.55},"bloom":{"tokens":29,"fertility":4.2,"p90":6.1,"cpt":1.01,"neglect":1.64}}}},{"code":"tir_Ethi","name":"Tigrinya","script":"Ethiopic","family":"Afro-Asiatic","region":"Africa","speakers":7507780,"floor":83,"floors":{"flores":83},"metrics":{"flores":{"gpt2":{"tokens":195,"fertility":7.8,"p90":10.11,"cpt":0.43,"neglect":1.35},"cl100k":{"tokens":194,"fertility":7.79,"p90":10.02,"cpt":0.43,"neglect":1.34},"o200k":{"tokens":147,"fertility":5.95,"p90":7.69,"cpt":0.57,"neglect":0.77},"llama3":{"tokens":194,"fertility":7.79,"p90":10,"cpt":0.43,"neglect":1.34},"gemma":{"tokens":83,"fertility":3.39,"p90":4.39,"cpt":0.99,"neglect":0},"tekken":{"tokens":194,"fertility":7.65,"p90":9.86,"cpt":0.43,"neglect":1.34},"qwen2":{"tokens":105,"fertility":4.17,"p90":5.38,"cpt":0.79,"neglect":0.27},"bloom":{"tokens":129,"fertility":5.12,"p90":6.55,"cpt":0.65,"neglect":0.55}}}},{"code":"tpi_Latn","name":"Tok Pisin","script":"Latin","family":"Indo-European","region":"Papunesia","speakers":4000000,"floor":44,"floors":{"flores":44},"metrics":{"flores":{"gpt2":{"tokens":55,"fertility":2.21,"p90":2.87,"cpt":2.82,"neglect":0.25},"cl100k":{"tokens":50,"fertility":2.03,"p90":2.64,"cpt":3.04,"neglect":0.14},"o200k":{"tokens":44,"fertility":1.75,"p90":2.26,"cpt":3.53,"neglect":0},"llama3":{"tokens":50,"fertility":2,"p90":2.61,"cpt":3.07,"neglect":0.14},"gemma":{"tokens":46,"fertility":1.84,"p90":2.42,"cpt":3.32,"neglect":0.05},"tekken":{"tokens":50,"fertility":1.96,"p90":2.53,"cpt":3.07,"neglect":0.14},"qwen2":{"tokens":51,"fertility":2,"p90":2.64,"cpt":3.02,"neglect":0.16},"bloom":{"tokens":48,"fertility":1.9,"p90":2.43,"cpt":3.22,"neglect":0.09}}}},{"code":"tsn_Latn","name":"Tswana","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":4500000,"floor":48,"floors":{"flores":48},"metrics":{"flores":{"gpt2":{"tokens":60,"fertility":2.41,"p90":3.07,"cpt":2.55,"neglect":0.25},"cl100k":{"tokens":57,"fertility":2.29,"p90":2.96,"cpt":2.66,"neglect":0.19},"o200k":{"tokens":48,"fertility":1.93,"p90":2.46,"cpt":3.2,"neglect":0},"llama3":{"tokens":57,"fertility":2.29,"p90":2.94,"cpt":2.67,"neglect":0.19},"gemma":{"tokens":53,"fertility":2.1,"p90":2.67,"cpt":2.91,"neglect":0.1},"tekken":{"tokens":56,"fertility":2.2,"p90":2.81,"cpt":2.7,"neglect":0.17},"qwen2":{"tokens":58,"fertility":2.27,"p90":2.94,"cpt":2.64,"neglect":0.21},"bloom":{"tokens":51,"fertility":2.03,"p90":2.58,"cpt":2.98,"neglect":0.06}}}},{"code":"tso_Latn","name":"Tsonga","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":13000000,"floor":42,"floors":{"flores":42},"metrics":{"flores":{"gpt2":{"tokens":60,"fertility":2.44,"p90":3.13,"cpt":2.39,"neglect":0.43},"cl100k":{"tokens":56,"fertility":2.25,"p90":2.88,"cpt":2.58,"neglect":0.33},"o200k":{"tokens":42,"fertility":1.71,"p90":2.19,"cpt":3.38,"neglect":0},"llama3":{"tokens":55,"fertility":2.22,"p90":2.86,"cpt":2.61,"neglect":0.31},"gemma":{"tokens":52,"fertility":2.08,"p90":2.67,"cpt":2.79,"neglect":0.24},"tekken":{"tokens":54,"fertility":2.14,"p90":2.76,"cpt":2.63,"neglect":0.29},"qwen2":{"tokens":56,"fertility":2.22,"p90":2.87,"cpt":2.56,"neglect":0.33},"bloom":{"tokens":50,"fertility":2,"p90":2.56,"cpt":2.88,"neglect":0.19}}}},{"code":"tuk_Latn","name":"Turkmen","script":"Latin","family":"Turkic","region":"Eurasia","speakers":16000000,"floor":40,"floors":{"flores":40},"metrics":{"flores":{"gpt2":{"tokens":68,"fertility":2.79,"p90":3.55,"cpt":1.84,"neglect":0.7},"cl100k":{"tokens":59,"fertility":2.39,"p90":3,"cpt":2.15,"neglect":0.47},"o200k":{"tokens":40,"fertility":1.65,"p90":2.03,"cpt":3.14,"neglect":0},"llama3":{"tokens":53,"fertility":2.12,"p90":2.64,"cpt":2.41,"neglect":0.33},"gemma":{"tokens":50,"fertility":2.02,"p90":2.57,"cpt":2.54,"neglect":0.25},"tekken":{"tokens":53,"fertility":2.08,"p90":2.6,"cpt":2.41,"neglect":0.33},"qwen2":{"tokens":55,"fertility":2.19,"p90":2.75,"cpt":2.31,"neglect":0.38},"bloom":{"tokens":54,"fertility":2.19,"p90":2.71,"cpt":2.34,"neglect":0.35}}}},{"code":"tum_Latn","name":"Tumbuka","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":7000000,"floor":51,"floors":{"flores":51},"metrics":{"flores":{"gpt2":{"tokens":68,"fertility":2.71,"p90":3.69,"cpt":2.3,"neglect":0.33},"cl100k":{"tokens":63,"fertility":2.53,"p90":3.45,"cpt":2.48,"neglect":0.24},"o200k":{"tokens":51,"fertility":2.05,"p90":2.75,"cpt":3.06,"neglect":0},"llama3":{"tokens":63,"fertility":2.5,"p90":3.42,"cpt":2.5,"neglect":0.24},"gemma":{"tokens":55,"fertility":2.2,"p90":3.04,"cpt":2.81,"neglect":0.08},"tekken":{"tokens":61,"fertility":2.38,"p90":3.23,"cpt":2.56,"neglect":0.2},"qwen2":{"tokens":63,"fertility":2.46,"p90":3.38,"cpt":2.49,"neglect":0.24},"bloom":{"tokens":54,"fertility":2.13,"p90":2.94,"cpt":2.88,"neglect":0.06}}}},{"code":"tur_Latn","name":"Turkish","script":"Latin","family":"Turkic","region":"Eurasia","speakers":82231620,"floor":34,"floors":{"flores":34,"massive":9},"metrics":{"flores":{"gpt2":{"tokens":60,"fertility":2.4,"p90":3.07,"cpt":2.07,"neglect":0.76},"cl100k":{"tokens":47,"fertility":1.9,"p90":2.43,"cpt":2.61,"neglect":0.38},"o200k":{"tokens":35,"fertility":1.41,"p90":1.73,"cpt":3.55,"neglect":0.03},"llama3":{"tokens":35,"fertility":1.4,"p90":1.71,"cpt":3.58,"neglect":0.03},"gemma":{"tokens":34,"fertility":1.39,"p90":1.71,"cpt":3.59,"neglect":0},"tekken":{"tokens":37,"fertility":1.47,"p90":1.84,"cpt":3.29,"neglect":0.09},"qwen2":{"tokens":41,"fertility":1.61,"p90":2.01,"cpt":3.04,"neglect":0.21},"bloom":{"tokens":48,"fertility":1.95,"p90":2.45,"cpt":2.53,"neglect":0.41}},"massive":{"gpt2":{"tokens":15,"fertility":2.17,"p90":3.2,"cpt":2.13,"neglect":0.67},"cl100k":{"tokens":12,"fertility":1.75,"p90":2.58,"cpt":2.69,"neglect":0.33},"o200k":{"tokens":9,"fertility":1.38,"p90":2,"cpt":3.49,"neglect":0},"llama3":{"tokens":9,"fertility":1.33,"p90":2,"cpt":3.52,"neglect":0},"gemma":{"tokens":9,"fertility":1.29,"p90":2,"cpt":3.72,"neglect":0},"tekken":{"tokens":10,"fertility":1.5,"p90":2,"cpt":3.13,"neglect":0.11},"qwen2":{"tokens":10,"fertility":1.5,"p90":2.25,"cpt":3.08,"neglect":0.11},"bloom":{"tokens":12,"fertility":1.71,"p90":2.5,"cpt":2.7,"neglect":0.33}}}},{"code":"twi_Latn","name":"Twi","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":9000000,"floor":46,"floors":{"flores":46},"metrics":{"flores":{"gpt2":{"tokens":65,"fertility":2.57,"p90":3.57,"cpt":1.83,"neglect":0.41},"cl100k":{"tokens":63,"fertility":2.47,"p90":3.47,"cpt":1.89,"neglect":0.37},"o200k":{"tokens":46,"fertility":1.83,"p90":2.5,"cpt":2.59,"neglect":0},"llama3":{"tokens":63,"fertility":2.47,"p90":3.46,"cpt":1.9,"neglect":0.37},"gemma":{"tokens":51,"fertility":2,"p90":2.78,"cpt":2.35,"neglect":0.11},"tekken":{"tokens":62,"fertility":2.36,"p90":3.3,"cpt":1.94,"neglect":0.35},"qwen2":{"tokens":55,"fertility":2.12,"p90":2.95,"cpt":2.18,"neglect":0.2},"bloom":{"tokens":46,"fertility":1.8,"p90":2.4,"cpt":2.61,"neglect":0}}}},{"code":"tzm_Tfng","name":"Central Moroccan Berber","script":"Tifinagh","family":"Afro-Asiatic","region":"Africa","speakers":17000000,"floor":121,"floors":{"flores":121},"metrics":{"flores":{"gpt2":{"tokens":254,"fertility":10.29,"p90":12.86,"cpt":0.42,"neglect":1.1},"cl100k":{"tokens":246,"fertility":10,"p90":12.56,"cpt":0.43,"neglect":1.03},"o200k":{"tokens":246,"fertility":10.1,"p90":12.6,"cpt":0.43,"neglect":1.03},"llama3":{"tokens":246,"fertility":10,"p90":12.56,"cpt":0.43,"neglect":1.03},"gemma":{"tokens":121,"fertility":4.91,"p90":6.26,"cpt":0.88,"neglect":0},"tekken":{"tokens":247,"fertility":9.82,"p90":12.35,"cpt":0.43,"neglect":1.04},"qwen2":{"tokens":126,"fertility":5,"p90":6.3,"cpt":0.85,"neglect":0.04},"bloom":{"tokens":189,"fertility":7.59,"p90":9.44,"cpt":0.56,"neglect":0.56}}}},{"code":"uig_Arab","name":"Uighur","script":"Arabic","family":"Turkic","region":"Eurasia","speakers":10400000,"floor":61,"floors":{"flores":61},"metrics":{"flores":{"gpt2":{"tokens":176,"fertility":7.08,"p90":9.16,"cpt":0.73,"neglect":1.89},"cl100k":{"tokens":129,"fertility":5.16,"p90":6.69,"cpt":1,"neglect":1.11},"o200k":{"tokens":61,"fertility":2.48,"p90":3.17,"cpt":2.09,"neglect":0},"llama3":{"tokens":110,"fertility":4.38,"p90":5.74,"cpt":1.18,"neglect":0.8},"gemma":{"tokens":85,"fertility":3.41,"p90":4.48,"cpt":1.51,"neglect":0.39},"tekken":{"tokens":106,"fertility":4.1,"p90":5.42,"cpt":1.23,"neglect":0.74},"qwen2":{"tokens":94,"fertility":3.71,"p90":4.82,"cpt":1.37,"neglect":0.54},"bloom":{"tokens":93,"fertility":3.64,"p90":4.68,"cpt":1.41,"neglect":0.52}}}},{"code":"ukr_Cyrl","name":"Ukrainian","script":"Cyrillic","family":"Indo-European","region":"Eurasia","speakers":26900000,"floor":41,"floors":{"flores":41},"metrics":{"flores":{"gpt2":{"tokens":142,"fertility":5.74,"p90":7.23,"cpt":0.87,"neglect":2.46},"cl100k":{"tokens":74,"fertility":2.95,"p90":3.72,"cpt":1.67,"neglect":0.8},"o200k":{"tokens":43,"fertility":1.74,"p90":2.12,"cpt":2.87,"neglect":0.05},"llama3":{"tokens":41,"fertility":1.64,"p90":2.03,"cpt":3.01,"neglect":0},"gemma":{"tokens":41,"fertility":1.66,"p90":2.06,"cpt":2.99,"neglect":0},"tekken":{"tokens":44,"fertility":1.73,"p90":2.15,"cpt":2.79,"neglect":0.07},"qwen2":{"tokens":64,"fertility":2.5,"p90":3.13,"cpt":1.95,"neglect":0.56},"bloom":{"tokens":68,"fertility":2.73,"p90":3.42,"cpt":1.8,"neglect":0.66}}}},{"code":"umb_Latn","name":"Umbundu","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":6000000,"floor":43,"floors":{"flores":43},"metrics":{"flores":{"gpt2":{"tokens":55,"fertility":2.21,"p90":2.93,"cpt":2.22,"neglect":0.28},"cl100k":{"tokens":50,"fertility":2,"p90":2.62,"cpt":2.46,"neglect":0.16},"o200k":{"tokens":43,"fertility":1.76,"p90":2.32,"cpt":2.82,"neglect":0},"llama3":{"tokens":48,"fertility":1.95,"p90":2.57,"cpt":2.51,"neglect":0.12},"gemma":{"tokens":46,"fertility":1.86,"p90":2.45,"cpt":2.66,"neglect":0.07},"tekken":{"tokens":48,"fertility":1.89,"p90":2.5,"cpt":2.53,"neglect":0.12},"qwen2":{"tokens":49,"fertility":1.97,"p90":2.57,"cpt":2.46,"neglect":0.14},"bloom":{"tokens":43,"fertility":1.73,"p90":2.28,"cpt":2.82,"neglect":0}}}},{"code":"urd_Arab","name":"Urdu","script":"Arabic","family":"Indo-European","region":"Eurasia","speakers":68600000,"floor":34,"floors":{"flores":34,"massive":9},"metrics":{"flores":{"gpt2":{"tokens":152,"fertility":6.24,"p90":7.76,"cpt":0.77,"neglect":3.47},"cl100k":{"tokens":108,"fertility":4.38,"p90":5.55,"cpt":1.1,"neglect":2.18},"o200k":{"tokens":40,"fertility":1.67,"p90":2,"cpt":2.93,"neglect":0.18},"llama3":{"tokens":77,"fertility":3.1,"p90":4,"cpt":1.56,"neglect":1.26},"gemma":{"tokens":47,"fertility":1.93,"p90":2.43,"cpt":2.49,"neglect":0.38},"tekken":{"tokens":45,"fertility":1.76,"p90":2.17,"cpt":2.66,"neglect":0.32},"qwen2":{"tokens":79,"fertility":3.18,"p90":4,"cpt":1.49,"neglect":1.32},"bloom":{"tokens":34,"fertility":1.38,"p90":1.67,"cpt":3.5,"neglect":0}},"massive":{"gpt2":{"tokens":43,"fertility":6.17,"p90":8.71,"cpt":0.76,"neglect":3.78},"cl100k":{"tokens":30,"fertility":4.38,"p90":6.2,"cpt":1.09,"neglect":2.33},"o200k":{"tokens":11,"fertility":1.67,"p90":2.4,"cpt":2.9,"neglect":0.22},"llama3":{"tokens":21,"fertility":3,"p90":4.38,"cpt":1.55,"neglect":1.33},"gemma":{"tokens":13,"fertility":2,"p90":2.86,"cpt":2.44,"neglect":0.44},"tekken":{"tokens":13,"fertility":1.86,"p90":2.71,"cpt":2.5,"neglect":0.44},"qwen2":{"tokens":22,"fertility":3.25,"p90":4.56,"cpt":1.47,"neglect":1.44},"bloom":{"tokens":9,"fertility":1.29,"p90":1.9,"cpt":3.57,"neglect":0}}}},{"code":"uzn_Latn","name":"Northern Uzbek","script":"Latin","family":"Turkic","region":"Eurasia","speakers":25100000,"floor":43,"floors":{"flores":43},"metrics":{"flores":{"gpt2":{"tokens":57,"fertility":2.29,"p90":2.9,"cpt":2.4,"neglect":0.33},"cl100k":{"tokens":53,"fertility":2.17,"p90":2.74,"cpt":2.54,"neglect":0.23},"o200k":{"tokens":43,"fertility":1.77,"p90":2.21,"cpt":3.14,"neglect":0},"llama3":{"tokens":52,"fertility":2.12,"p90":2.67,"cpt":2.6,"neglect":0.21},"gemma":{"tokens":49,"fertility":2,"p90":2.52,"cpt":2.78,"neglect":0.14},"tekken":{"tokens":46,"fertility":1.82,"p90":2.28,"cpt":2.95,"neglect":0.07},"qwen2":{"tokens":54,"fertility":2.16,"p90":2.73,"cpt":2.52,"neglect":0.26},"bloom":{"tokens":49,"fertility":2,"p90":2.46,"cpt":2.75,"neglect":0.14}}}},{"code":"vec_Latn","name":"Venetian","script":"Latin","family":"Indo-European","region":"Eurasia","speakers":2000000,"floor":37,"floors":{"flores":37},"metrics":{"flores":{"gpt2":{"tokens":49,"fertility":1.97,"p90":2.46,"cpt":2.47,"neglect":0.32},"cl100k":{"tokens":42,"fertility":1.68,"p90":2.09,"cpt":2.89,"neglect":0.14},"o200k":{"tokens":38,"fertility":1.52,"p90":1.87,"cpt":3.22,"neglect":0.03},"llama3":{"tokens":42,"fertility":1.67,"p90":2.08,"cpt":2.9,"neglect":0.14},"gemma":{"tokens":37,"fertility":1.48,"p90":1.82,"cpt":3.3,"neglect":0},"tekken":{"tokens":39,"fertility":1.53,"p90":1.92,"cpt":3.08,"neglect":0.05},"qwen2":{"tokens":42,"fertility":1.66,"p90":2.07,"cpt":2.88,"neglect":0.14},"bloom":{"tokens":39,"fertility":1.55,"p90":1.91,"cpt":3.1,"neglect":0.05}}}},{"code":"vie_Latn","name":"Vietnamese","script":"Latin","family":"Austroasiatic","region":"Eurasia","speakers":76000000,"floor":32,"floors":{"flores":32,"massive":9},"metrics":{"flores":{"gpt2":{"tokens":112,"fertility":4.57,"p90":5.93,"cpt":1.13,"neglect":2.5},"cl100k":{"tokens":60,"fertility":2.43,"p90":3.16,"cpt":2.1,"neglect":0.88},"o200k":{"tokens":36,"fertility":1.46,"p90":1.82,"cpt":3.49,"neglect":0.12},"llama3":{"tokens":34,"fertility":1.37,"p90":1.68,"cpt":3.7,"neglect":0.06},"gemma":{"tokens":34,"fertility":1.35,"p90":1.68,"cpt":3.75,"neglect":0.06},"tekken":{"tokens":36,"fertility":1.41,"p90":1.74,"cpt":3.51,"neglect":0.12},"qwen2":{"tokens":36,"fertility":1.42,"p90":1.75,"cpt":3.52,"neglect":0.12},"bloom":{"tokens":32,"fertility":1.26,"p90":1.53,"cpt":4.01,"neglect":0}},"massive":{"gpt2":{"tokens":31,"fertility":4.4,"p90":6.57,"cpt":1.15,"neglect":2.44},"cl100k":{"tokens":18,"fertility":2.62,"p90":4,"cpt":1.94,"neglect":1},"o200k":{"tokens":11,"fertility":1.67,"p90":3,"cpt":3,"neglect":0.22},"llama3":{"tokens":11,"fertility":1.55,"p90":2.8,"cpt":3.13,"neglect":0.22},"gemma":{"tokens":10,"fertility":1.5,"p90":2.5,"cpt":3.4,"neglect":0.11},"tekken":{"tokens":12,"fertility":1.62,"p90":3.5,"cpt":2.74,"neglect":0.33},"qwen2":{"tokens":10,"fertility":1.45,"p90":2,"cpt":3.57,"neglect":0.11},"bloom":{"tokens":9,"fertility":1.33,"p90":2,"cpt":3.74,"neglect":0}}}},{"code":"war_Latn","name":"Waray (Philippines)","script":"Latin","family":"Austronesian","region":"Papunesia","speakers":3100000,"floor":42,"floors":{"flores":42},"metrics":{"flores":{"gpt2":{"tokens":58,"fertility":2.37,"p90":3,"cpt":2.57,"neglect":0.38},"cl100k":{"tokens":49,"fertility":1.95,"p90":2.43,"cpt":3.12,"neglect":0.17},"o200k":{"tokens":42,"fertility":1.72,"p90":2.11,"cpt":3.57,"neglect":0},"llama3":{"tokens":48,"fertility":1.93,"p90":2.39,"cpt":3.15,"neglect":0.14},"gemma":{"tokens":44,"fertility":1.79,"p90":2.2,"cpt":3.42,"neglect":0.05},"tekken":{"tokens":47,"fertility":1.86,"p90":2.29,"cpt":3.21,"neglect":0.12},"qwen2":{"tokens":49,"fertility":1.94,"p90":2.42,"cpt":3.09,"neglect":0.17},"bloom":{"tokens":45,"fertility":1.79,"p90":2.22,"cpt":3.35,"neglect":0.07}}}},{"code":"wol_Latn","name":"Wolof","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":3700000,"floor":42,"floors":{"flores":42},"metrics":{"flores":{"gpt2":{"tokens":52,"fertility":2.09,"p90":2.75,"cpt":2.2,"neglect":0.24},"cl100k":{"tokens":47,"fertility":1.88,"p90":2.47,"cpt":2.45,"neglect":0.12},"o200k":{"tokens":42,"fertility":1.7,"p90":2.2,"cpt":2.75,"neglect":0},"llama3":{"tokens":46,"fertility":1.86,"p90":2.43,"cpt":2.49,"neglect":0.1},"gemma":{"tokens":43,"fertility":1.71,"p90":2.23,"cpt":2.7,"neglect":0.02},"tekken":{"tokens":46,"fertility":1.81,"p90":2.38,"cpt":2.48,"neglect":0.1},"qwen2":{"tokens":47,"fertility":1.86,"p90":2.45,"cpt":2.44,"neglect":0.12},"bloom":{"tokens":42,"fertility":1.65,"p90":2.1,"cpt":2.77,"neglect":0}}}},{"code":"xho_Latn","name":"Xhosa","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":8177300,"floor":40,"floors":{"flores":40},"metrics":{"flores":{"gpt2":{"tokens":56,"fertility":2.26,"p90":2.89,"cpt":2.29,"neglect":0.4},"cl100k":{"tokens":51,"fertility":2.05,"p90":2.62,"cpt":2.5,"neglect":0.28},"o200k":{"tokens":40,"fertility":1.64,"p90":2.06,"cpt":3.18,"neglect":0},"llama3":{"tokens":51,"fertility":2.03,"p90":2.59,"cpt":2.53,"neglect":0.28},"gemma":{"tokens":47,"fertility":1.89,"p90":2.41,"cpt":2.72,"neglect":0.17},"tekken":{"tokens":50,"fertility":1.97,"p90":2.5,"cpt":2.54,"neglect":0.25},"qwen2":{"tokens":51,"fertility":2.03,"p90":2.61,"cpt":2.49,"neglect":0.28},"bloom":{"tokens":41,"fertility":1.67,"p90":2.09,"cpt":3.07,"neglect":0.03}}}},{"code":"ydd_Hebr","name":"Eastern Yiddish","script":"Hebrew","family":"Indo-European","region":"Eurasia","speakers":1000000,"floor":49,"floors":{"flores":49},"metrics":{"flores":{"gpt2":{"tokens":162,"fertility":6.61,"p90":8.1,"cpt":0.8,"neglect":2.31},"cl100k":{"tokens":135,"fertility":5.54,"p90":6.8,"cpt":0.94,"neglect":1.76},"o200k":{"tokens":49,"fertility":2,"p90":2.43,"cpt":2.61,"neglect":0},"llama3":{"tokens":135,"fertility":5.54,"p90":6.8,"cpt":0.94,"neglect":1.76},"gemma":{"tokens":64,"fertility":2.61,"p90":3.19,"cpt":2,"neglect":0.31},"tekken":{"tokens":75,"fertility":2.94,"p90":3.64,"cpt":1.73,"neglect":0.53},"qwen2":{"tokens":69,"fertility":2.76,"p90":3.36,"cpt":1.87,"neglect":0.41},"bloom":{"tokens":109,"fertility":4.41,"p90":5.26,"cpt":1.18,"neglect":1.22}}}},{"code":"yor_Latn","name":"Yoruba","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":37800000,"floor":41,"floors":{"flores":41},"metrics":{"flores":{"gpt2":{"tokens":93,"fertility":3.88,"p90":5.69,"cpt":1.23,"neglect":1.27},"cl100k":{"tokens":72,"fertility":2.94,"p90":4.16,"cpt":1.61,"neglect":0.76},"o200k":{"tokens":52,"fertility":2.16,"p90":3.05,"cpt":2.2,"neglect":0.27},"llama3":{"tokens":66,"fertility":2.7,"p90":3.77,"cpt":1.75,"neglect":0.61},"gemma":{"tokens":59,"fertility":2.42,"p90":3.41,"cpt":1.94,"neglect":0.44},"tekken":{"tokens":69,"fertility":2.73,"p90":4.1,"cpt":1.64,"neglect":0.68},"qwen2":{"tokens":66,"fertility":2.68,"p90":3.74,"cpt":1.74,"neglect":0.61},"bloom":{"tokens":41,"fertility":1.6,"p90":2.18,"cpt":2.84,"neglect":0}}}},{"code":"yue_Hant","name":"Yue Chinese","script":"Han (Traditional)","family":"Sino-Tibetan","region":"Eurasia","speakers":73100000,"floor":22,"floors":{"flores":22},"metrics":{"flores":{"gpt2":{"tokens":75,"fertility":3.03,"p90":3.76,"cpt":0.49,"neglect":2.41},"cl100k":{"tokens":51,"fertility":2.06,"p90":2.59,"cpt":0.71,"neglect":1.32},"o200k":{"tokens":34,"fertility":1.35,"p90":1.7,"cpt":1.09,"neglect":0.55},"llama3":{"tokens":32,"fertility":1.29,"p90":1.61,"cpt":1.13,"neglect":0.45},"gemma":{"tokens":26,"fertility":1.06,"p90":1.32,"cpt":1.37,"neglect":0.18},"tekken":{"tokens":36,"fertility":1.41,"p90":1.77,"cpt":1.01,"neglect":0.64},"qwen2":{"tokens":28,"fertility":1.09,"p90":1.36,"cpt":1.31,"neglect":0.27},"bloom":{"tokens":22,"fertility":0.89,"p90":1.12,"cpt":1.61,"neglect":0}}}},{"code":"zho_Hans","name":"Chinese (Simplified)","script":"Han (Simplified)","family":"Sino-Tibetan","region":"Eurasia","speakers":941000000,"floor":23,"floors":{"flores":23,"massive":6},"metrics":{"flores":{"gpt2":{"tokens":80,"fertility":3.17,"p90":3.98,"cpt":0.51,"neglect":2.48},"cl100k":{"tokens":46,"fertility":1.86,"p90":2.41,"cpt":0.87,"neglect":1},"o200k":{"tokens":30,"fertility":1.23,"p90":1.63,"cpt":1.31,"neglect":0.3},"llama3":{"tokens":31,"fertility":1.27,"p90":1.65,"cpt":1.26,"neglect":0.35},"gemma":{"tokens":26,"fertility":1.05,"p90":1.41,"cpt":1.49,"neglect":0.13},"tekken":{"tokens":36,"fertility":1.39,"p90":1.81,"cpt":1.11,"neglect":0.57},"qwen2":{"tokens":25,"fertility":1,"p90":1.32,"cpt":1.57,"neglect":0.09},"bloom":{"tokens":23,"fertility":0.91,"p90":1.2,"cpt":1.74,"neglect":0}},"massive":{"gpt2":{"tokens":21,"fertility":3,"p90":4.33,"cpt":0.48,"neglect":2.5},"cl100k":{"tokens":12,"fertility":1.7,"p90":2.5,"cpt":0.84,"neglect":1},"o200k":{"tokens":7,"fertility":1.08,"p90":1.57,"cpt":1.33,"neglect":0.17},"llama3":{"tokens":8,"fertility":1.17,"p90":1.67,"cpt":1.23,"neglect":0.33},"gemma":{"tokens":6,"fertility":0.94,"p90":1.33,"cpt":1.59,"neglect":0},"tekken":{"tokens":9,"fertility":1.29,"p90":2,"cpt":1.08,"neglect":0.5},"qwen2":{"tokens":6,"fertility":0.88,"p90":1.29,"cpt":1.64,"neglect":0},"bloom":{"tokens":6,"fertility":0.82,"p90":1.17,"cpt":1.72,"neglect":0}}}},{"code":"zho_Hant","name":"Chinese (Traditional)","script":"Han (Traditional)","family":"Sino-Tibetan","region":"Eurasia","speakers":941000000,"floor":23,"floors":{"flores":23,"massive":6},"metrics":{"flores":{"gpt2":{"tokens":78,"fertility":3.11,"p90":3.86,"cpt":0.49,"neglect":2.39},"cl100k":{"tokens":53,"fertility":2.12,"p90":2.67,"cpt":0.71,"neglect":1.3},"o200k":{"tokens":35,"fertility":1.41,"p90":1.76,"cpt":1.08,"neglect":0.52},"llama3":{"tokens":33,"fertility":1.33,"p90":1.68,"cpt":1.12,"neglect":0.43},"gemma":{"tokens":27,"fertility":1.11,"p90":1.4,"cpt":1.36,"neglect":0.17},"tekken":{"tokens":37,"fertility":1.47,"p90":1.82,"cpt":1,"neglect":0.61},"qwen2":{"tokens":28,"fertility":1.14,"p90":1.43,"cpt":1.29,"neglect":0.22},"bloom":{"tokens":23,"fertility":0.94,"p90":1.18,"cpt":1.59,"neglect":0}},"massive":{"gpt2":{"tokens":19,"fertility":2.8,"p90":4.33,"cpt":0.48,"neglect":2.17},"cl100k":{"tokens":13,"fertility":2,"p90":3,"cpt":0.7,"neglect":1.17},"o200k":{"tokens":8,"fertility":1.25,"p90":1.86,"cpt":1.13,"neglect":0.33},"llama3":{"tokens":8,"fertility":1.22,"p90":1.83,"cpt":1.13,"neglect":0.33},"gemma":{"tokens":6,"fertility":1,"p90":1.4,"cpt":1.48,"neglect":0},"tekken":{"tokens":9,"fertility":1.38,"p90":2,"cpt":0.99,"neglect":0.5},"qwen2":{"tokens":7,"fertility":1,"p90":1.5,"cpt":1.39,"neglect":0.17},"bloom":{"tokens":6,"fertility":0.83,"p90":1.25,"cpt":1.62,"neglect":0}}}},{"code":"zsm_Latn","name":"Standard Malay","script":"Latin","family":"Austronesian","region":"Papunesia","speakers":33000000,"floor":27,"floors":{"flores":27,"massive":8},"metrics":{"flores":{"gpt2":{"tokens":51,"fertility":2.06,"p90":2.55,"cpt":2.66,"neglect":0.89},"cl100k":{"tokens":41,"fertility":1.61,"p90":2,"cpt":3.35,"neglect":0.52},"o200k":{"tokens":32,"fertility":1.29,"p90":1.58,"cpt":4.24,"neglect":0.19},"llama3":{"tokens":40,"fertility":1.61,"p90":1.97,"cpt":3.37,"neglect":0.48},"gemma":{"tokens":29,"fertility":1.17,"p90":1.43,"cpt":4.62,"neglect":0.07},"tekken":{"tokens":34,"fertility":1.33,"p90":1.65,"cpt":3.94,"neglect":0.26},"qwen2":{"tokens":41,"fertility":1.61,"p90":2,"cpt":3.32,"neglect":0.52},"bloom":{"tokens":27,"fertility":1.07,"p90":1.3,"cpt":5.01,"neglect":0}},"massive":{"gpt2":{"tokens":14,"fertility":2,"p90":2.75,"cpt":2.66,"neglect":0.75},"cl100k":{"tokens":11,"fertility":1.6,"p90":2.25,"cpt":3.38,"neglect":0.38},"o200k":{"tokens":9,"fertility":1.33,"p90":1.88,"cpt":4.09,"neglect":0.12},"llama3":{"tokens":11,"fertility":1.58,"p90":2.2,"cpt":3.41,"neglect":0.38},"gemma":{"tokens":8,"fertility":1.2,"p90":1.71,"cpt":4.54,"neglect":0},"tekken":{"tokens":10,"fertility":1.4,"p90":2,"cpt":3.74,"neglect":0.25},"qwen2":{"tokens":11,"fertility":1.6,"p90":2.25,"cpt":3.38,"neglect":0.38},"bloom":{"tokens":8,"fertility":1.17,"p90":1.67,"cpt":4.57,"neglect":0}}}},{"code":"zul_Latn","name":"Zulu","script":"Latin","family":"Atlantic-Congo","region":"Africa","speakers":12100000,"floor":42,"floors":{"flores":42},"metrics":{"flores":{"gpt2":{"tokens":60,"fertility":2.38,"p90":3,"cpt":2.28,"neglect":0.43},"cl100k":{"tokens":55,"fertility":2.19,"p90":2.72,"cpt":2.48,"neglect":0.31},"o200k":{"tokens":42,"fertility":1.7,"p90":2.08,"cpt":3.23,"neglect":0},"llama3":{"tokens":54,"fertility":2.16,"p90":2.7,"cpt":2.51,"neglect":0.29},"gemma":{"tokens":50,"fertility":1.98,"p90":2.47,"cpt":2.73,"neglect":0.19},"tekken":{"tokens":53,"fertility":2.08,"p90":2.59,"cpt":2.54,"neglect":0.26},"qwen2":{"tokens":55,"fertility":2.16,"p90":2.7,"cpt":2.46,"neglect":0.31},"bloom":{"tokens":43,"fertility":1.75,"p90":2.12,"cpt":3.09,"neglect":0.02}}}}]}