Whisper large for 51 Afriacn languages

This model is an adaptation of whisper-large-v3 for 51 languages widely spoken in Africa: English, French, Luganda, Acholi, Lugbara, Ateso, Runyankole, Rutooro, Lumasaba, Swahili, Lusoga, Kinyarwanda, Rukiga, Afrikaans, Akan, Amharic, Bambara, Bemba, Berber, Chichewa, Dagaare, Dagbani, Ewe, French, Fulani, Hausa, Igbo, Ikposo, Kabyle, Kalenjin, Kanuri, Kikuyu, Kwamba, Lendu, Lingala, Luhya, Luo, Malagasy, Ndebele, Nigerian Pidgin, Oromo, Rukonjo, Ruruuli, Shona, Somali, Sotho, Thur, Tswana, Wolof, Xhosa, Yoruba, Zulu.

Usage

The model is used in a similar way to the base Whisper model. To perform the transcription for certain language, for better accuracy, please include the language token mapping SALT_LANGUAGE_TOKENS_WHISPER, and specify the language code in the forced_decoder_ids during generation. See the example below:

import transformers
import datasets
import torch

model_id = "huwenjie333/whisper-v3-ft-af51"
processor = transformers.WhisperProcessor.from_pretrained(model_id)
model = transformers.WhisperForConditionalGeneration.from_pretrained(model_id)

SALT_LANGUAGE_TOKENS_WHISPER = {
    # Existing languges codes from Whisper
    "eng": 50259,
    "fra": 50265,
    "swa": 50318,
    "sna": 50324,
    "yor": 50325,
    "som": 50326,
    "afr": 50327,
    "amh": 50334,
    "mlg": 50349,
    "lin": 50353,
    "hau": 50354,
    # Overwrite unused language tokens
    "ach": 50357,
    "aka": 50356,
    "bam": 50355,
    "bem": 50352,
    "ber": 50351,
    "cgg": 50350,
    "dag": 50348,
    "dga": 50347,
    "ewe": 50346,
    "ful": 50345,
    "ibo": 50344,
    "kab": 50343,
    "kau": 50342,
    "kik": 50341,
    "kin": 50340,
    "kln": 50339,
    "koo": 50338,
    "kpo": 50337,
    "led": 50336,
    "lgg": 50335,
    "lth": 50333,
    "lug": 50332,
    "luo": 50331,
    "luy": 50330,
    "myx": 50329,
    "nbl": 50328,
    "nya": 50323,
    "nyn": 50322,
    "orm": 50321,
    "pcm": 50320,
    "ruc": 50319,
    "rwm": 50317,
    "sot": 50316,
    "teo": 50315,
    "tsn": 50314,
    "ttj": 50313,
    "wol": 50312,
    "xho": 50311,
    "xog": 50310,
    "zul": 50309,
}

# Get some test audio
ds = datasets.load_dataset('Sunbird/salt', 'multispeaker-lug', split='test')
audio = ds[0]['audio']
sample_rate = ds[0]['sample_rate']

# Specify a language from one of the above.
lang = 'lug'

# Apply the model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
input_features = processor(
    audio, sampling_rate=sample_rate, return_tensors="pt").input_features
input_features = input_features.to(device)

lang_tok = SALT_LANGUAGE_TOKENS_WHISPER[lang_code]
transcribe_tok = processor.tokenizer.convert_tokens_to_ids("<|transcribe|>")
notimestamps_tok = processor.tokenizer.convert_tokens_to_ids("<|notimestamps|>")
forced_decoder_ids = [
    (1, lang_tok),
    (2, transcribe_tok),
    (3, notimestamps_tok),
]

predicted_ids = model.to(device).generate(
    input_features,
    forced_decoder_ids=forced_decoder_ids,
    num_beams=1,
    do_sample=False,
)
transcription = processor.batch_decode(
  predicted_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)

print(transcription)
# Ekikoola kya kasooli kya kyenvu wabula langi yaakyo etera okuba eya kitaka wansi.

Performance Metrics

code   language            WER      CER    n_examples
afr    Afrikaans         0.082    0.029    170
aka    Akan              0.328    0.133     50
amh    Amharic           0.403    0.266    250
bam    Bambara           0.361    0.181    149
bem    Bemba             0.401    0.087    100
ber    Berber            0.655    0.513    100
cgg    Rukiga            0.245    0.055     49
dag    Dagbani           0.306    0.097    100
dga    Dagaare           0.299    0.119     50
ewe    Ewe               0.289    0.095     78
fra    French            0.019    0.012     50
ful    Fulani            0.391    0.111    149
hau    Hausa             0.174    0.049    287
ibo    Igbo              0.338    0.116    247
kab    Kabyle            0.439    0.213     50
kau    Kanuri            0.436    0.113     50
kik    Kikuyu            0.210    0.098     50
kln    Kalenjin          0.497    0.118    100
koo    Rukonjo           0.532    0.096     47
kpo    Ikposo            0.629    0.234     50
led    Lendu             0.272    0.093     50
lin    Lingala           0.238    0.097    193
lth    Thur              0.152    0.060     41
luo    Luo               0.210    0.051    149
luy    Luhya             1.138    0.839     50
mlg    Malagasy          0.146    0.060     49
nbl    Ndebele           0.250    0.045     10
nya    Chichewa          0.264    0.055     98
orm    Oromo             0.242    0.063    119
pcm    Nigerian Pidgin   0.157    0.061    100
ruc    Ruruuli           0.492    0.105     38
rwm    Kwamba            0.577    0.217     49
sna    Shona             0.198    0.051    147
som    Somali            0.397    0.134    149
sot    Sotho             0.285    0.166     94
tsn    Tswana            0.074    0.021     98
ttj    Rutooro           0.234    0.069     49
wol    Wolof             0.329    0.123    135
xho    Xhosa             0.264    0.052    126
yor    Yoruba            0.367    0.128    249
zul    Zulu              0.203    0.040    143
ach    Acholi            0.200    0.045    101
eng    English           0.045    0.018    101
lgg    Lugbara           0.205    0.049    101
lug    Luganda           0.105    0.020    103
nyn    Runyankole        0.248    0.050    103
teo    Ateso             0.272    0.066    101
myx    Lumasaba          0.368    0.078    100
xog    Lusoga            0.366    0.067    100
swa    Swahili           0.070    0.020     25
kin    Kinyarwanda       0.210    0.072     25