symbols: support group references in replacements

devDocs/developerGuide.t2t

@@ -72,6 +72,7 @@ Subsequent lines contain a textual identifier used to identify the symbol, a tab
 For example:
 ```
 . sentence ending	(?<=[^\s.])\.(?=[\"')\s]|$)
+dates with .	\b(\d\d)\.(\d\d)\.(\d{2}|\d{4})\b
 ```
 
 Again, the English symbols are inherited by all other locales, so you need not include any complex symbols already defined for English.
@@ -98,6 +99,8 @@ Certain characters cannot be typed into the file, so the following special seque
  - \f: form feed
  - \#: # character (needed because # at the start of a line denotes a comment)
 - replacement: The text which should be spoken for the symbol.
+If the symbol is a complex symbol, \1, \2, etc. can be used to refer to the groups matches, which will be inlined in the replacement, allowing for simpler rules.
+This also means that to get a \ character in the replacement, one has to type \\.
 - level: The symbol level at which the symbol should be spoken.
 The symbol level is configured by the user and specifies the amount of symbols that should be spoken.
 This field should contain one of the levels "none", "some", "most", "all" or "char", or "-" to use the default.
@@ -133,6 +136,13 @@ It means that the ". sentence ending" complex symbol should be spoken as "point"
 Level and preserve are not specified, so they will be taken from English.
 A display name is provided so that French users will know what the symbol represents.
 
+```
+dates with .	\1 point \2 point \3	all	norep	# date avec points
+```
+This line appears in the French symbols.dic file.
+It means that the first, second, and third groups of the match will be included, separated by the word 'point'.
+The effect is thus to replace the dots from the date with the word 'point'.
+
 Please see the file locale\en\symbols.dic for the English definitions which are inherited for all locales.
 This is also a good full example.
 

source/characterProcessing.py

@@ -519,6 +519,35 @@ def __init__(self, locale):
 			log.error("Invalid complex symbol regular expression in locale %s: %s" % (locale, e))
 			raise LookupError
 
+	def _replaceGroups(self, m: re.Match, string: str) -> str:
+		"""Replace matching group references (\\1, \\2, ...) with the corresponding matched groups.
+		Also replace \\\\ with \\ and reject other escapes, for escaping coherency.
+		@param m: The currently-matched group
+		@param string: The match replacement string which may contain group references
+		"""
+		result = ''
+
+		in_escape = False
+		for char in string:
+			if not in_escape:
+				if char == '\\':
+					in_escape = True
+				else:
+					result += char
+			else:
+				if char == '\\':
+					result += '\\'
+				elif char >= '0' and char <= '9':
+					result += m.group(m.lastindex + ord(char) - ord('0'))
+				else:
+					log.error("Invalid reference \\%string" % char)
+					raise LookupError
+				in_escape = False
+		if in_escape:
+			log.error("Unterminated backslash")
+			raise LookupError
+		return result
+
 	def _regexpRepl(self, m):
 		group = m.lastgroup
 
@@ -540,16 +569,19 @@ def _regexpRepl(self, m):
 			if group == "simple":
 				# Simple symbol.
 				symbol = self.computedSymbols[text]
+				replacement = symbol.replacement
 			else:
 				# Complex symbol.
 				index = int(group[1:])
 				symbol = self._computedComplexSymbolsList[index]
+				replacement = self._replaceGroups(m, symbol.replacement)
+
 			if symbol.preserve == SYMPRES_ALWAYS or (symbol.preserve == SYMPRES_NOREP and self._level < symbol.level):
 				suffix = text
 			else:
 				suffix = " "
-			if self._level >= symbol.level and symbol.replacement:
-				return u" {repl}{suffix}".format(repl=symbol.replacement, suffix=suffix)
+			if self._level >= symbol.level and replacement:
+				return u" {repl}{suffix}".format(repl=replacement, suffix=suffix)
 			else:
 				return suffix
 

tests/unit/test_characterProcessing.py

@@ -0,0 +1,120 @@
+# A part of NonVisual Desktop Access (NVDA)
+# This file is covered by the GNU General Public License.
+# See the file COPYING for more details.
+# Copyright (C) 2020 NV Access Limited
+
+"""Unit tests for the characterProcessing module.
+"""
+
+import unittest
+import re
+from characterProcessing import SpeechSymbolProcessor
+
+
+class TestComplex(unittest.TestCase):
+	"""Test the complex symbols rules.
+	"""
+
+	def _replace_cb(self, replacement, name=None):
+		"""Return a regexp callback which replaces matches of the given
+		group name (or all groups if no name is given) with the
+		replacement string, with support for replacement of group
+		references.
+		"""
+		def replace(m):
+			if name is None or m.lastgroup == name:
+				return SpeechSymbolProcessor._replaceGroups(self, m, replacement)
+			return m.group()
+		return replace
+
+	def _replace(self, string, pattern, replacement, name=None):
+		"""Perform a pattern-based replacement on a string, for the
+		given named group (or all groups if no name is given), with
+		support for replacement of group references.
+		"""
+		regexp = re.compile(pattern, re.UNICODE)
+		return regexp.sub(self._replace_cb(replacement, name), string)
+
+	def test_group_replacement(self):
+		"""Test that plain text gets properly replaced
+		"""
+		replaced = self._replace(
+			string="1",
+			pattern=r"(\d)",
+			replacement="a"
+		)
+		self.assertEqual(replaced, "a")
+
+	def test_backslash_replacement(self):
+		"""Test that backslashes get properly replaced
+		"""
+		replaced = self._replace(
+			string="1",
+			pattern=r"(\d)",
+			replacement=r"\\"
+		)
+		self.assertEqual(replaced, "\\")
+
+	def test_double_backslash_replacement(self):
+		"""Test that double backslashes get properly replaced
+		"""
+		replaced = self._replace(
+			string="1",
+			pattern=r"(\d)",
+			replacement=r"\\\\"
+		)
+		self.assertEqual(replaced, r"\\")
+
+	def test_unknown_escape(self):
+		"""Test that a non-supported escaped character (i.e. not \\1,
+		\\2, ... \\9 and \\\\) in the replacement raises an error
+		"""
+		with self.assertRaises(LookupError):
+			self._replace(
+				string="1",
+				pattern=r"(\d)",
+				replacement=r"\a"
+			)
+
+	def test_missing_group(self):
+		"""Test that a reference in the replacement to an non-existing
+		group raises an error
+		"""
+		with self.assertRaises(IndexError):
+			self._replace(
+				string="1",
+				pattern=r"(\d)",
+				replacement=r"\2"
+			)
+
+	def test_unterminated_escape(self):
+		"""Test that an escape at the end of replacement raises an
+		error, since there is nothing to be escaped there
+		"""
+		with self.assertRaises(LookupError):
+			self._replace(
+				string="1",
+				pattern=r"(\d)",
+				replacement="\\"
+			)
+
+	def test_group_replacements(self):
+		"""Test that group references get properly replaced
+		"""
+		replaced = self._replace(
+			string="bar.BAT",
+			pattern=r"(([a-z]*)\.([A-Z]*))",
+			replacement=r"\2>\1"
+		)
+		self.assertEqual(replaced, "BAT>bar")
+
+	def test_multiple_group_replacement(self):
+		"""Test that group indexing is correct with multiple groups
+		"""
+		replaced = self._replace(
+			string="bar.BAT",
+			pattern=r"(baz)|(?P<foo>([a-z]*)\.([A-Z]*))",
+			replacement=r"\2>\1",
+			name="foo"
+		)
+		self.assertEqual(replaced, "BAT>bar")

user_docs/en/changes.t2t

@@ -8,6 +8,7 @@ What's New in NVDA
 == New Features ==
 - Pressing F1 inside NVDA dialogs will now open the help file to most relevant section. (#7757)
 - Support for auto complete suggestions (IntelliSense) Microsoft SQL Server Management Studio and Visual Studio 2017. (#7504)
+- Symbol pronunciation: Support for grouping in a complex symbol definition and support group references in a replacement rule making them simpler and more powerful. (#11107)
 
 
 == Changes ==

user_docs/en/userGuide.t2t

@@ -1922,6 +1922,9 @@ You can remove a symbol you previously added by pressing the Remove button.
 
 When you are finished, press the OK button to save your changes or the Cancel button to discard them.
 
+In the case of complex symbols, the Replacement field may have to include some group references of the matched text. For instance, for a pattern matching a whole date, \1, \2, and \3 would need to appear in the field, to be replaced by the corresponding parts of the date.
+Normal backslashes in the Replacement field should thus be doubled, e.g. "a\\b" should be typed in order to get the "a\b" replacement.
+
 +++ Input Gestures +++[InputGestures]
 In this dialog, you can customize the input gestures (keys on the keyboard, buttons on a braille display, etc.) for NVDA commands.