Patch wx overriding the correct python locale

source/core.py

@@ -447,6 +447,8 @@ def handlePowerStatusChange(self):
 	if wxLang:
 		try:
 			locale.Init(wxLang.Language)
+			# Revert wx's changes to the python locale
+			languageHandler.setLocale(languageHandler.curLang)
 		except:
 			log.error("Failed to initialize wx locale",exc_info=True)
 

source/languageHandler.py

@@ -13,6 +13,7 @@
 import locale
 import gettext
 import globalVars
+from logHandler import log
 
 #a few Windows locale constants
 LOCALE_SLANGUAGE=0x2
@@ -146,37 +147,109 @@ def getWindowsLanguage():
 			localeName="en"
 	return localeName
 
-def setLanguage(lang):
+
+def setLanguage(lang: str) -> None:
+	'''
+	Sets the following using `lang`
+	 - languageHandler.curLang
+	 - the translation service (fallback to English)
+	 - the windows locale for the thread (fallback to system locale)
+	 - the python locale for the thread (match the translation service, fallback to python default)
+	'''
 	global curLang
+	if lang == "Windows":
+		localeName = getWindowsLanguage()
+	else:
+		localeName = lang
 	try:
-		if lang=="Windows":
-			localeName=getWindowsLanguage()
-			trans=gettext.translation('nvda',localedir='locale',languages=[localeName])
-			curLang=localeName
-		else:
-			trans=gettext.translation("nvda", localedir="locale", languages=[lang])
-			curLang=lang
-			localeChanged=False
-			#Try setting Python's locale to lang
-			try:
-				locale.setlocale(locale.LC_ALL,lang)
-				localeChanged=True
-			except:
-				pass
-			if not localeChanged and '_' in lang:
-				#Python couldn'tsupport the language_country locale, just try language.
-				try:
-					locale.setlocale(locale.LC_ALL,lang.split('_')[0])
-				except:
-					pass
-			#Set the windows locale for this thread (NVDA core) to this locale.
-			LCID=localeNameToWindowsLCID(lang)
-			ctypes.windll.kernel32.SetThreadLocale(LCID)
+		trans = gettext.translation("nvda", localedir="locale", languages=[localeName])
+		curLang = localeName
 	except IOError:
-		trans=gettext.translation("nvda",fallback=True)
-		curLang="en"
+		log.debugWarning(f"couldn't set the translation service locale to {localeName}")
+		trans = gettext.translation("nvda", fallback=True)
+		curLang = "en"
+
+	# Set the windows locale for this thread (NVDA core) to this locale.
+	if lang != "Windows":
+		try:
+			LCID = localeNameToWindowsLCID(localeName)
+			ctypes.windll.kernel32.SetThreadLocale(LCID)
+		except IOError:
+			log.debugWarning(f"couldn't set windows thread locale to {lang}")
+
 	# #9207: Python 3.8 adds gettext.pgettext, so add it to the built-in namespace.
 	trans.install(names=["pgettext"])
+	setLocale(curLang)
+
+
+def setLocale(localeName: str) -> None:
+	'''
+	Set python's locale using a `localeName` set by `setLanguage`.
+	Will fallback on `curLang` if it cannot be set and finally fallback to the system locale.
+	Python 3.8's locale system allows you to set locales that you cannot get
+	so we must test for both ValueErrors and locale.Errors
+	'''
+
+	r'''
+	>>> import locale
+	>>> locale.setlocale(locale.LC_ALL, 'foobar')
+	Traceback (most recent call last):
+	File "<stdin>", line 1, in <module>
+	File "Python38-32\lib\locale.py", line 608, in setlocale
+		return _setlocale(category, locale)
+	locale.Error: unsupported locale setting
+	>>> locale.setlocale(locale.LC_ALL, 'en-GB')
+	'en-GB'
+	>>> locale.getlocale()
+	Traceback (most recent call last):
+	File "<stdin>", line 1, in <module>
+	File "Python38-32\lib\locale.py", line 591, in getlocale
+		return _parse_localename(localename)
+	File "Python38-32\lib\locale.py", line 499, in _parse_localename
+		raise ValueError('unknown locale: %s' % localename)
+	ValueError: unknown locale: en-GB
+	'''
+	originalLocaleName = localeName
+	# Try setting Python's locale to localeName
+	try:
+		locale.setlocale(locale.LC_ALL, localeName)
+		locale.getlocale()
+		return
+	except (locale.Error, ValueError):
+		pass
+	if '-' in localeName:
+		# Python couldn't support the language-country locale, try language_country.
+		try:
+			localeName = localeName.replace('-', '_')
+			locale.setlocale(locale.LC_ALL, localeName)
+			locale.getlocale()
+			return
+		except (locale.Error, ValueError):
+			pass
+	if '_' in localeName:
+		# Python couldn't support the language_country locale, just try language.
+		try:
+			localeName = localeName.split('_')[0]
+			locale.setlocale(locale.LC_ALL, localeName)
+			locale.getlocale()
+			return
+		except (locale.Error, ValueError):
+			pass
+
+	log.debugWarning(f"python locale {localeName} could not be set")
+	try:
+		locale.getlocale()
+	except ValueError:
+		# as the locale may have been changed to something that getlocale() couldn't retrieve
+		# reset to default locale
+		if originalLocaleName == curLang:
+			log.debugWarning(f"setting python locale to system default")
+			# reset to system locale default if we can't set the current lang's locale
+			locale.setlocale(locale.LC_ALL, "")
+		else:
+			log.debugWarning(f"setting python locale to {curLang}")
+			# fallback and try to reset the locale to the current lang
+			setLocale(curLang)
 
 
 def getLanguage() -> str:
@@ -314,5 +387,9 @@ def normalizeLanguage(lang):
 	134:'qut',
 	135:'rw',
 	136:'wo',
-	140:'gbz'
+	140: 'gbz',
+	1170: 'ckb',
+	1109: 'my',
+	1143: 'so',
+	9242: 'sr',
 }

tests/unit/test_languageHandler.py

@@ -1,20 +1,87 @@
-#tests/unit/test_languageHandler.py
-#A part of NonVisual Desktop Access (NVDA)
-#This file is covered by the GNU General Public License.
-#See the file COPYING for more details.
-#Copyright (C) 2017 NV Access Limited
+# A part of NonVisual Desktop Access (NVDA)
+# This file is covered by the GNU General Public License.
+# See the file COPYING for more details.
+# Copyright (C) 2017-2021 NV Access Limited
 
 """Unit tests for the languageHandler module.
 """
 
 import unittest
 import languageHandler
-from languageHandler import LCID_NONE
+from languageHandler import LCID_NONE, windowsPrimaryLCIDsToLocaleNames
+import locale
+import ctypes
 
 LCID_ENGLISH_US = 0x0409
+UNSUPPORTED_PYTHON_LOCALES = {
+	"an",
+	"arn",
+	"arn_CL",
+	"ba",
+	"ba_RU",
+	"bn",
+	"bo",
+	"bo_BT",
+	"ckb",
+	"co",
+	"co_FR",
+	"en_BZ",
+	"en_CB",
+	"en_JA",
+	"en_MY",
+	"en_TT",
+	"en_US",
+	"fil",
+	"fy",
+	"gbz",
+	"gbz_AF",
+	"gu",
+	"ha",
+	"hy",
+	"ii",
+	"ii_CN",
+	"kh",
+	"kh_KH",
+	"kk",
+	"kmr",
+	"kok",
+	"lb",
+	"mn",
+	"mn_CN",
+	"moh",
+	"moh_CA",
+	"my",
+	"ne",
+	"ns",
+	"ns_ZA",
+	"ps",
+	"qut",
+	"qut_GT",
+	"quz",
+	"quz_BO",
+	"quz_EC",
+	"rm",
+	"rm_CH",
+	"sa",
+	"se_FI",
+	"se_SE",
+	"so",
+	"sw",
+	"tk",
+	"tmz",
+	"tmz_DZ",
+	"ug",
+	"wen",
+	"wen_DE",
+	"wo",
+	"yo",
+}
+TRANSLATABLE_LANGS = set(l[0] for l in languageHandler.getAvailableLanguages()) - {"Windows"}
+WINDOWS_LANGS = set(locale.windows_locale.values())
+WINDOWS_LANGS.update(windowsPrimaryLCIDsToLocaleNames.values())
 
-class TestLocaleNameToWindowsLCID(unittest.TestCase):
 
+class TestLocaleNameToWindowsLCID(unittest.TestCase):
 	def test_knownLocale(self):
 		lcid = languageHandler.localeNameToWindowsLCID("en")
 		self.assertEqual(lcid, LCID_ENGLISH_US)
@@ -31,3 +98,119 @@ def test_nonStandardLocale(self):
 	def test_invalidLocale(self):
 		lcid = languageHandler.localeNameToWindowsLCID("zzzz")
 		self.assertEqual(lcid, LCID_NONE)
+
+
+class TestSetLocale(unittest.TestCase):
+	"""
+	tests setting the python locale for all possible locales set by NVDA or the System
+	"""
+
+	SUPPORTED_LOCALES = [("en", "en_US"), ("fa-IR", "fa_IR"), ("an-ES", "an_ES")]
+
+	@classmethod
+	def tearDownClass(cls):
+		languageHandler.setLanguage(languageHandler.curLang)
+
+	def setUp(self):
+		languageHandler.setLanguage(languageHandler.curLang)
+
+	def testSupportedLocales(self):
+		for localeName in self.SUPPORTED_LOCALES:
+			languageHandler.setLocale(localeName[0])
+			self.assertEqual(locale.getlocale()[0], localeName[1])
+
+	def testUnsupportedLocales(self):
+		original_locale = locale.getlocale()
+		for localeName in UNSUPPORTED_PYTHON_LOCALES:
+			with self.subTest():
+				languageHandler.setLocale(localeName)
+				self.assertEqual(locale.getlocale(), original_locale)
+
+	def testAllSupportedLangs(self):
+		for localeName in TRANSLATABLE_LANGS - UNSUPPORTED_PYTHON_LOCALES:
+			with self.subTest():
+				languageHandler.setLocale(localeName)
+				current_locale = locale.getlocale()
+
+				if localeName == "uk":
+					self.assertEqual(current_locale[0], "English_United Kingdom")
+				else:
+					pythonLang = current_locale[0].split("_")[0]
+					langOnly = localeName.split("_")[0]
+					self.assertEqual(
+						langOnly,
+						pythonLang,
+						f"full values: {localeName} {current_locale[0]}",
+					)
+
+	def testAllWindowsLangs(self):
+		prev_locale = locale.getlocale()
+		for localeName in WINDOWS_LANGS:
+			with self.subTest():
+				languageHandler.setLocale(localeName)
+				current_locale = locale.getlocale()
+				if localeName == languageHandler.curLang:
+					self.assertEqual(current_locale, prev_locale)
+				elif localeName in UNSUPPORTED_PYTHON_LOCALES:
+					self.assertEqual(current_locale, prev_locale)
+				else:
+					self.assertNotEqual(current_locale, prev_locale, localeName)
+
+
+class TestSetLanguage(unittest.TestCase):
+	"""
+	tests setting the NVDA language for all possible locales set by NVDA
+	"""
+	UNSUPPORTED_WIN_LANGUAGES = ["an", "kmr"]
+
+	def tearDown(self):
+		languageHandler.setLanguage(self._prevLang)
+
+	def __init__(self, *args, **kwargs):
+		self._prevLang = languageHandler.getLanguage()
+
+		ctypes.windll.kernel32.SetThreadLocale(0)
+		defaultThreadLocale = ctypes.windll.kernel32.GetThreadLocale()
+		self._defaultThreadLocaleName = languageHandler.windowsLCIDToLocaleName(
+			defaultThreadLocale
+		)
+
+		locale.setlocale(locale.LC_ALL, "")
+		self._defaultPythonLocale = locale.getlocale()
+
+		languageHandler.setLanguage(self._prevLang)
+		super().__init__(*args, **kwargs)
+
+	def testTranslatableLanguages(self):
+		for lang in TRANSLATABLE_LANGS:
+			langOnly = lang.split("_")[0]
+			with self.subTest():
+				languageHandler.setLanguage(lang)
+				# check curLang/translation service is set
+				self.assertEqual(languageHandler.curLang, lang)
+
+				# check Windows thread is set
+				threadLocale = ctypes.windll.kernel32.GetThreadLocale()
+				threadLocaleName = languageHandler.windowsLCIDToLocaleName(threadLocale)
+				threadLocaleLang = threadLocaleName.split("_")[0]
+				if lang in self.UNSUPPORTED_WIN_LANGUAGES:
+					self.assertEqual(self._defaultThreadLocaleName, threadLocaleName)
+				else:
+					# check that the language codes are correctly set
+					self.assertEqual(
+						langOnly,
+						threadLocaleLang,
+						f"full values: {lang} {threadLocaleName}",
+					)
+
+				# check that the python locale is set
+				python_locale = locale.getlocale()
+				if lang in UNSUPPORTED_PYTHON_LOCALES:
+					self.assertEqual(self._defaultPythonLocale, python_locale)
+				elif lang == "uk":
+					self.assertEqual(python_locale[0], "English_United Kingdom")
+				else:
+					pythonLang = python_locale[0].split("_")[0]
+					self.assertEqual(
+						langOnly, pythonLang, f"full values: {lang} {python_locale}"
+					)