[whisper] support arbitrary language and task

[xingchensong]

use https://github.com/saffsd/langid.py to detect language on the fly.

results look good:

import langid
print(langid.classify("Questo è un testo di prova."))
print(langid.classify("hello world"))
print(langid.classify("中国北京天安门"))
print(langid.classify("使用GPS进行定位"))
print(langid.classify("Hi, 你好"))

('it', -95.88065242767334)
('en', -23.719746112823486)                                                                                                                                                                                                          ('zh', -90.74214363098145)
('zh', -78.27495694160461)
('zh', -35.134679317474365)

langid could detect most of the common languages.

from whisper.tokenizer import LANGUAGES as WhiserLanguages
WHISPER_LANGS = tuple(WhiserLanguages.keys())

all_langs = "af, am, an, ar, as, az, be, bg, bn, br, bs, ca, cs, cy, da, de, dz, el, en, eo, es, et, eu, fa, fi, fo, fr, ga, gl, gu, he, hi, hr, ht, hu, hy, id, is, it, ja, jv, ka, kk, km, kn, ko, ku, ky, la, lb, lo, lt, lv, mg, mk, ml, mn, mr, ms, mt, nb, ne, nl, nn, no, oc, or, pa, pl, ps, pt, qu, ro, ru, rw, se, si, sk, sl, sq, sr, sv, sw, ta, te, th, tl, tr, ug, uk, ur, vi, vo, wa, xh, zh, zu"
all_langs = all_langs.split()
all_langs = [i.replace(",", "") for i in all_langs]

l_in_whisper, l_not_in_whisper = [], []
for l in all_langs:
    if l not in WHISPER_LANGS:
        l_not_in_whisper.append(l)
    else:
        l_in_whisper.append(l)

print(l_not_in_whisper)
print(l_in_whisper)

l_not_in_whisper: ['an', 'dz', 'eo', 'ga', 'jv', 'ku', 'ky', 'nb', 'or', 'qu', 'rw', 'se', 'ug', 'vo', 'wa', 'xh', 'zu']
l_in_whisper: ['af', 'am', 'ar', 'as', 'az', 'be', 'bg', 'bn', 'br', 'bs', 'ca', 'cs', 'cy', 'da', 'de', 'el', 'en', 'es', 'et', 'eu', 'fa', 'fi', 'fo', 'fr', 'gl', 'gu', 'he', 'hi', 'hr', 'ht', 'hu', 'hy', 'id', 'is', 'it', 'ja', 'ka', 'kk', 'km', 'kn', 'ko', 'la', 'lb', 'lo', 'lt', 'lv', 'mg', 'mk', 'ml', 'mn', 'mr', 'ms', 'mt', 'ne', 'nl', 'nn', 'no', 'oc', 'pa', 'pl', 'ps', 'pt', 'ro', 'ru', 'si', 'sk', 'sl', 'sq', 'sr', 'sv', 'sw', 'ta', 'te', 'th', 'tl', 'tr', 'uk', 'ur', 'vi', 'zh']