Handling the new _xlws. prefix used for SORT() and FILTER() functions in both the Calculation Engine, and in the Xlsx Writer

MarkBaker · MarkBaker · commit 5839533dc139 · 2022-12-15T13:16:45.000+01:00
Create stubs for new uncategorised ANCHORARRAY(), LAMBDA(), and SINGLE(); and for the new Logical functions BYCOL() and BYROW()
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -31,6 +31,7 @@ and this project adheres to [Semantic Versioning](https://semver.org).
 
 ### Fixed
 
+- Fixed handling for `_xlws` prefixed functions from Office365 [Issue #3245](https://github.com/PHPOffice/PhpSpreadsheet/issues/3245) [PR #3247](https://github.com/PHPOffice/PhpSpreadsheet/pull/3247)
 - Conditionals formatting rules applied to rows/columns are removed [Issue #3184](https://github.com/PHPOffice/PhpSpreadsheet/issues/3184) [PR #3213](https://github.com/PHPOffice/PhpSpreadsheet/pull/3213)
 - Treat strings containing currency or accounting values as floats in Calculation Engine operations [Issue #3165](https://github.com/PHPOffice/PhpSpreadsheet/issues/3165) [PR #3189](https://github.com/PHPOffice/PhpSpreadsheet/pull/3189)
 - Treat strings containing percentage values as floats in Calculation Engine operations [Issue #3155](https://github.com/PHPOffice/PhpSpreadsheet/issues/3155) [PR #3156](https://github.com/PHPOffice/PhpSpreadsheet/pull/3156) and [PR #3164](https://github.com/PHPOffice/PhpSpreadsheet/pull/3164)
diff --git a/src/PhpSpreadsheet/Calculation/Calculation.php b/src/PhpSpreadsheet/Calculation/Calculation.php
@@ -33,7 +33,7 @@ class Calculation
     //    Opening bracket
     const CALCULATION_REGEXP_OPENBRACE = '\(';
     //    Function (allow for the old @ symbol that could be used to prefix a function, but we'll ignore it)
-    const CALCULATION_REGEXP_FUNCTION = '@?(?:_xlfn\.)?([\p{L}][\p{L}\p{N}\.]*)[\s]*\(';
+    const CALCULATION_REGEXP_FUNCTION = '@?(?:_xlfn\.)?(?:_xlws\.)?([\p{L}][\p{L}\p{N}\.]*)[\s]*\(';
     //    Cell reference (cell or range of cells, with or without a sheet reference)
     const CALCULATION_REGEXP_CELLREF = '((([^\s,!&%^\/\*\+<>=:`-]*)|(\'(?:[^\']|\'[^!])+?\')|(\"(?:[^\"]|\"[^!])+?\"))!)?\$?\b([a-z]{1,3})\$?(\d{1,7})(?![\w.])';
     //    Cell reference (with or without a sheet reference) ensuring absolute/relative
@@ -328,6 +328,11 @@ public static function getExcelConstants(string $key)
             'functionCall' => [MathTrig\Arabic::class, 'evaluate'],
             'argumentCount' => '1',
         ],
+        'ANCHORARRAY' => [
+            'category' => Category::CATEGORY_UNCATEGORISED,
+            'functionCall' => [Functions::class, 'DUMMY'],
+            'argumentCount' => '*',
+        ],
         'AREAS' => [
             'category' => Category::CATEGORY_LOOKUP_AND_REFERENCE,
             'functionCall' => [Functions::class, 'DUMMY'],
@@ -503,6 +508,16 @@ public static function getExcelConstants(string $key)
             'functionCall' => [Engineering\BitWise::class, 'BITRSHIFT'],
             'argumentCount' => '2',
         ],
+        'BYCOL' => [
+            'category' => Category::CATEGORY_LOGICAL,
+            'functionCall' => [Functions::class, 'DUMMY'],
+            'argumentCount' => '*',
+        ],
+        'BYROW' => [
+            'category' => Category::CATEGORY_LOGICAL,
+            'functionCall' => [Functions::class, 'DUMMY'],
+            'argumentCount' => '*',
+        ],
         'CEILING' => [
             'category' => Category::CATEGORY_MATH_AND_TRIG,
             'functionCall' => [MathTrig\Ceiling::class, 'ceiling'],
@@ -1613,6 +1628,11 @@ public static function getExcelConstants(string $key)
             'functionCall' => [Statistical\Deviations::class, 'kurtosis'],
             'argumentCount' => '1+',
         ],
+        'LAMBDA' => [
+            'category' => Category::CATEGORY_UNCATEGORISED,
+            'functionCall' => [Functions::class, 'DUMMY'],
+            'argumentCount' => '*',
+        ],
         'LARGE' => [
             'category' => Category::CATEGORY_STATISTICAL,
             'functionCall' => [Statistical\Size::class, 'large'],
@@ -2307,6 +2327,11 @@ public static function getExcelConstants(string $key)
             'functionCall' => [MathTrig\Trig\Sine::class, 'sin'],
             'argumentCount' => '1',
         ],
+        'SINGLE' => [
+            'category' => Category::CATEGORY_UNCATEGORISED,
+            'functionCall' => [Functions::class, 'DUMMY'],
+            'argumentCount' => '*',
+        ],
         'SINH' => [
             'category' => Category::CATEGORY_MATH_AND_TRIG,
             'functionCall' => [MathTrig\Trig\Sine::class, 'sinh'],
diff --git a/src/PhpSpreadsheet/Calculation/Category.php b/src/PhpSpreadsheet/Calculation/Category.php
@@ -17,4 +17,5 @@ abstract class Category
     const CATEGORY_STATISTICAL = 'Statistical';
     const CATEGORY_TEXT_AND_DATA = 'Text and Data';
     const CATEGORY_WEB = 'Web';
+    const CATEGORY_UNCATEGORISED = 'Uncategorised';
 }
diff --git a/src/PhpSpreadsheet/Writer/Xlsx/FunctionPrefix.php b/src/PhpSpreadsheet/Writer/Xlsx/FunctionPrefix.php
@@ -2,9 +2,9 @@
 
 namespace PhpOffice\PhpSpreadsheet\Writer\Xlsx;
 
-class Xlfn
+class FunctionPrefix
 {
-    const XLFNREGEXP = '/(?<!_xlfn[.])\\b('
+    const XLFNREGEXP = '/(?:_xlfn\.)?((?:_xlws\.)?('
             // functions added with Excel 2010
         . 'beta[.]dist'
         . '|beta[.]inv'
@@ -134,6 +134,7 @@ class Xlfn
         // functions added with Excel 365
         . '|filter'
         . '|randarray'
+        . '|anchorarray'
         . '|sequence'
         . '|sort'
         . '|sortby'
@@ -143,27 +144,51 @@ class Xlfn
         . '|arraytotext'
         . '|call'
         . '|let'
+        . '|lambda'
+        . '|single'
         . '|register[.]id'
         . '|textafter'
         . '|textbefore'
         . '|textsplit'
         . '|valuetotext'
-        . ')(?=\\s*[(])/i';
+        . '))\s*\(/Umui';
+
+    const XLWSREGEXP = '/(?<!_xlws\.)('
+        // functions added with Excel 365
+        . 'filter'
+        . '|sort'
+        . ')\s*\(/mui';
+
+    /**
+     * Prefix function name in string with _xlfn. where required.
+     */
+    protected static function addXlfnPrefix(string $functionString): string
+    {
+        return (string) preg_replace(self::XLFNREGEXP, '_xlfn.$1(', $functionString);
+    }
+
+    /**
+     * Prefix function name in string with _xlws. where required.
+     */
+    protected static function addXlwsPrefix(string $functionString): string
+    {
+        return (string) preg_replace(self::XLWSREGEXP, '_xlws.$1(', $functionString);
+    }
 
     /**
      * Prefix function name in string with _xlfn. where required.
      */
-    public static function addXlfn(string $funcstring): string
+    public static function addFunctionPrefix(string $functionString): string
     {
-        return (string) preg_replace(self::XLFNREGEXP, '_xlfn.$1', $funcstring);
+        return self::addXlwsPrefix(self::addXlfnPrefix($functionString));
     }
 
     /**
      * Prefix function name in string with _xlfn. where required.
      * Leading character, expected to be equals sign, is stripped.
      */
-    public static function addXlfnStripEquals(string $funcstring): string
+    public static function addFunctionPrefixStripEquals(string $functionString): string
     {
-        return self::addXlfn(substr($funcstring, 1));
+        return self::addFunctionPrefix(substr($functionString, 1));
     }
 }
diff --git a/src/PhpSpreadsheet/Writer/Xlsx/Worksheet.php b/src/PhpSpreadsheet/Writer/Xlsx/Worksheet.php
@@ -498,7 +498,7 @@ private static function writeOtherCondElements(XMLWriter $objWriter, Conditional
                 if (is_bool($formula)) {
                     $formula = $formula ? 'TRUE' : 'FALSE';
                 }
-                $objWriter->writeElement('formula', Xlfn::addXlfn("$formula"));
+                $objWriter->writeElement('formula', FunctionPrefix::addFunctionPrefix("$formula"));
             }
         } else {
             if ($conditional->getConditionType() == Conditional::CONDITION_CONTAINSBLANKS) {
@@ -1204,7 +1204,7 @@ private function writeCellError(XMLWriter $objWriter, string $mappedType, string
     {
         $objWriter->writeAttribute('t', $mappedType);
         $cellIsFormula = substr($cellValue, 0, 1) === '=';
-        self::writeElementIf($objWriter, $cellIsFormula, 'f', Xlfn::addXlfnStripEquals($cellValue));
+        self::writeElementIf($objWriter, $cellIsFormula, 'f', FunctionPrefix::addFunctionPrefixStripEquals($cellValue));
         $objWriter->writeElement('v', $cellIsFormula ? $formulaerr : $cellValue);
     }
 
@@ -1234,7 +1234,7 @@ private function writeCellFormula(XMLWriter $objWriter, string $cellValue, Cell
             $objWriter->text(substr($cellValue, 1));
             $objWriter->endElement();
         } else {
-            $objWriter->writeElement('f', Xlfn::addXlfnStripEquals($cellValue));
+            $objWriter->writeElement('f', FunctionPrefix::addFunctionPrefixStripEquals($cellValue));
             self::writeElementIf(
                 $objWriter,
                 $this->getParentWriter()->getOffice2003Compatibility() === false,
diff --git a/tests/PhpSpreadsheetTests/DocumentGeneratorTest.php b/tests/PhpSpreadsheetTests/DocumentGeneratorTest.php
@@ -133,6 +133,11 @@ public function providerGenerateFunctionListByCategory(): array
                     Excel Function           | PhpSpreadsheet Function
                     -------------------------|--------------------------------------
 
+                    ## CATEGORY_UNCATEGORISED
+
+                    Excel Function           | PhpSpreadsheet Function
+                    -------------------------|--------------------------------------
+
                     EXPECTED
 
             ],
diff --git a/tests/PhpSpreadsheetTests/Writer/Xlsx/FunctionPrefixTest.php b/tests/PhpSpreadsheetTests/Writer/Xlsx/FunctionPrefixTest.php
@@ -0,0 +1,54 @@
+<?php
+
+namespace PhpOffice\PhpSpreadsheetTests\Writer\Xlsx;
+
+use PhpOffice\PhpSpreadsheet\Writer\Xlsx\FunctionPrefix;
+use PHPUnit\Framework\TestCase;
+
+class FunctionPrefixTest extends TestCase
+{
+    /**
+     * @dataProvider functionPrefixProvider
+     */
+    public function testFunctionPrefix(string $expectedResult, string $functionString): void
+    {
+        $result = FunctionPrefix::addFunctionPrefix($functionString);
+        self::assertSame($expectedResult, $result);
+    }
+
+    public function functionPrefixProvider(): array
+    {
+        return [
+            'Basic Legacy Function' => ['SUM()', 'SUM()'],
+            'New Function without Prefix' => ['_xlfn.ARABIC()', 'ARABIC()'],
+            'New Function already Prefixed' => ['_xlfn.ARABIC()', '_xlfn.ARABIC()'],
+            'New Function requiring Double-Prefix' => ['_xlfn._xlws.FILTER()', 'FILTER()'],
+            'New Function requiring Double-Prefix already partially Prefixed' => ['_xlfn._xlws.FILTER()', '_xlfn.FILTER()'],
+            'New Function requiring Double-Prefix already partially Prefixed #2' => ['_xlfn._xlws.FILTER()', '_xlws.FILTER()'],
+            'New Function requiring Double-Prefix already Fully Prefixed' => ['_xlfn._xlws.FILTER()', '_xlfn._xlws.FILTER()'],
+            'Multiple Functions' => ['_xlfn._xlws.SORT(_xlfn._xlws.FILTER(A:A, A:A<>""))', 'SORT(FILTER(A:A, A:A<>""))'],
+        ];
+    }
+
+//    /**
+//     * @dataProvider functionPrefixWithEqualsProvider
+//     */
+//    public function testFunctionPrefixWithEquals(string $expectedResult, string $functionString): void
+//    {
+//        $result = FunctionPrefix::addFunctionPrefixStripEquals($functionString);
+//        self::assertSame($expectedResult, $result);
+//    }
+//
+//    public function functionPrefixWithEqualsProvider(): array
+//    {
+//        return [
+//            'Basic Legacy Function' => ['SUM()', '=SUM()'],
+//            'New Function without Prefix' => ['_xlfn.ARABIC()', '=ARABIC()'],
+//            'New Function already Prefixed' => ['_xlfn.ARABIC()', '=_xlfn.ARABIC()'],
+//            'New Function requiring Double-Prefix' => ['_xlfn._xlws.FILTER()', '=FILTER()'],
+//            'New Function requiring Double-Prefix already partially Prefixed' => ['_xlfn._xlws.FILTER()', '=_xlfn.FILTER()'],
+//            'New Function requiring Double-Prefix already partially Prefixed #2' => ['_xlfn._xlws.FILTER()', '=_xlws.FILTER()'],
+//            'New Function requiring Double-Prefix already Fully Prefixed' => ['_xlfn._xlws.FILTER()', '=_xlfn._xlws.FILTER()'],
+//        ];
+//    }
+}

Original file line number	Diff line number	Diff line change
`@@ -17,4 +17,5 @@ abstract class Category`
`17`	`17`	`const CATEGORY_STATISTICAL = 'Statistical';`
`18`	`18`	`const CATEGORY_TEXT_AND_DATA = 'Text and Data';`
`19`	`19`	`const CATEGORY_WEB = 'Web';`
	`20`	`+ const CATEGORY_UNCATEGORISED = 'Uncategorised';`
`20`	`21`	`}`
Original file line number	Diff line number	Diff line change
`@@ -498,7 +498,7 @@ private static function writeOtherCondElements(XMLWriter $objWriter, Conditional`
`498`	`498`	`if (is_bool($formula)) {`
`499`	`499`	`$formula = $formula ? 'TRUE' : 'FALSE';`
`500`	`500`	`}`
`501`		`- $objWriter->writeElement('formula', Xlfn::addXlfn("$formula"));`
	`501`	`+ $objWriter->writeElement('formula', FunctionPrefix::addFunctionPrefix("$formula"));`
`502`	`502`	`}`
`503`	`503`	`} else {`
`504`	`504`	`if ($conditional->getConditionType() == Conditional::CONDITION_CONTAINSBLANKS) {`
`@@ -1204,7 +1204,7 @@ private function writeCellError(XMLWriter $objWriter, string $mappedType, string`
`1204`	`1204`	`{`
`1205`	`1205`	`$objWriter->writeAttribute('t', $mappedType);`
`1206`	`1206`	`$cellIsFormula = substr($cellValue, 0, 1) === '=';`
`1207`		`- self::writeElementIf($objWriter, $cellIsFormula, 'f', Xlfn::addXlfnStripEquals($cellValue));`
	`1207`	`+ self::writeElementIf($objWriter, $cellIsFormula, 'f', FunctionPrefix::addFunctionPrefixStripEquals($cellValue));`
`1208`	`1208`	`$objWriter->writeElement('v', $cellIsFormula ? $formulaerr : $cellValue);`
`1209`	`1209`	`}`
`1210`	`1210`
`@@ -1234,7 +1234,7 @@ private function writeCellFormula(XMLWriter $objWriter, string $cellValue, Cell`
`1234`	`1234`	`$objWriter->text(substr($cellValue, 1));`
`1235`	`1235`	`$objWriter->endElement();`
`1236`	`1236`	`} else {`
`1237`		`- $objWriter->writeElement('f', Xlfn::addXlfnStripEquals($cellValue));`
	`1237`	`+ $objWriter->writeElement('f', FunctionPrefix::addFunctionPrefixStripEquals($cellValue));`
`1238`	`1238`	`self::writeElementIf(`
`1239`	`1239`	`$objWriter,`
`1240`	`1240`	`$this->getParentWriter()->getOffice2003Compatibility() === false,`