Put back the old settings but deprecate them

package.json

@@ -1547,6 +1547,27 @@
                     "description": "Types to exclude from showing in the Interactive variable explorer",
                     "scope": "resource"
                 },
+                "jupyter.codeRegularExpression": {
+                    "type": "string",
+                    "deprecationMessage": "This setting is no longer being used. Use jupyter.codeLensExpressions instead",
+                    "default": "^(#\\s*%%|#\\s*\\<codecell\\>|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])",
+                    "description": "Regular expression used to identify code cells. All code until the next match is considered part of this cell.",
+                    "scope": "resource"
+                },
+                "jupyter.defaultCellMarker": {
+                    "type": "string",
+                    "deprecationMessage": "This setting is no longer being used. Use jupyter.codeLensExpressions instead",
+                    "default": "# %%",
+                    "description": "Cell marker used for delineating a cell in a python file.",
+                    "scope": "resource"
+                },
+                "jupyter.markdownRegularExpression": {
+                    "type": "string",
+                    "deprecationMessage": "This setting is no longer being used. Use jupyter.codeLensExpressions instead",
+                    "default": "^(#\\s*%%\\s*\\[markdown\\]|#\\s*\\<markdowncell\\>)",
+                    "description": "Regular expression used to identify markdown cells. All comments after this expression are considered part of the markdown.",
+                    "scope": "resource"
+                },
                 "jupyter.codeLensExpressions": {
                     "type": "array",
                     "description": "Language to code lens mapping for identifying markdown and code cells based on different languages or file extensions",

src/client/common/configSettings.ts

@@ -56,6 +56,8 @@ export class JupyterSettings implements IWatchableJupyterSettings {
     public maxOutputSize: number = -1;
     public enableScrollingForCellOutputs: boolean = false;
     public sendSelectionToInteractiveWindow: boolean = false;
+    public markdownRegularExpression: string = '';
+    public codeRegularExpression: string = '';
     public allowLiveShare: boolean = false;
     public errorBackgroundColor: string = '';
     public ignoreVscodeTheme: boolean = false;
@@ -80,6 +82,7 @@ export class JupyterSettings implements IWatchableJupyterSettings {
     public runMagicCommands: string = '';
     public runStartupCommands: string | string[] = [];
     public debugJustMyCode: boolean = false;
+    public defaultCellMarker: string = '';
     public themeMatplotlibPlots: boolean = false;
     public variableQueries: IVariableQuery[] = [];
     public codeLensExpressions: ICodeLensExpressions[] = [];

src/client/common/types.ts

@@ -133,6 +133,8 @@ export interface IJupyterSettings {
     readonly enableScrollingForCellOutputs: boolean;
     readonly sendSelectionToInteractiveWindow: boolean;
     readonly allowLiveShare: boolean;
+    readonly markdownRegularExpression: string | undefined;
+    readonly codeRegularExpression: string | undefined;
     readonly errorBackgroundColor: string;
     readonly ignoreVscodeTheme: boolean;
     readonly variableExplorerExclude: string;
@@ -157,6 +159,7 @@ export interface IJupyterSettings {
     readonly runStartupCommands: string | string[];
     readonly debugJustMyCode: boolean;
     readonly verboseLogging: boolean;
+    readonly defaultCellMarker: string | undefined;
     readonly themeMatplotlibPlots: boolean;
     readonly variableQueries: IVariableQuery[];
     readonly codeLensExpressions: ICodeLensExpressions[];

src/client/datascience/cellMatcher.ts

@@ -25,21 +25,37 @@ export class CellMatcher {
         const codeLens = settings?.codeLensExpressions.find
             ? settings?.codeLensExpressions.find((v) => v.language === language)
             : undefined;
+
+        // Use either the old or new settings based on if different and the language is python.
+        const codeExpression =
+            language == PYTHON_LANGUAGE &&
+            settings?.codeRegularExpression &&
+            settings?.codeRegularExpression !== RegExpValues.PythonCellMarker.source
+                ? settings?.codeRegularExpression
+                : codeLens?.codeExpression;
+        const markdownExpression =
+            language == PYTHON_LANGUAGE &&
+            settings?.markdownRegularExpression &&
+            settings?.markdownRegularExpression !== RegExpValues.PythonMarkdownCellMarker.source
+                ? settings?.markdownRegularExpression
+                : codeLens?.markdownExpression;
+        const defaultExpression =
+            language == PYTHON_LANGUAGE && settings?.defaultCellMarker && settings?.defaultCellMarker !== '# %%'
+                ? settings?.defaultCellMarker
+                : codeLens?.defaultCellMarker;
+
         const backupCellRegex =
             language == MARKDOWN_LANGUAGE ? RegExpValues.MarkdownCellMarker : RegExpValues.PythonCellMarker;
         const backupMarkdownRegex =
             language == MARKDOWN_LANGUAGE
                 ? RegExpValues.MarkdownMarkdownCellMarker
                 : RegExpValues.PythonMarkdownCellMarker;
 
-        this.codeMatchRegEx = this.createRegExp(codeLens ? codeLens.codeExpression : undefined, backupCellRegex);
-        this.markdownMatchRegEx = this.createRegExp(
-            codeLens ? codeLens.markdownExpression : undefined,
-            backupMarkdownRegex
-        );
+        this.codeMatchRegEx = this.createRegExp(codeExpression, backupCellRegex);
+        this.markdownMatchRegEx = this.createRegExp(markdownExpression, backupMarkdownRegex);
         this.codeExecRegEx = new RegExp(`${this.codeMatchRegEx.source}(.*)`);
         this.markdownExecRegEx = new RegExp(`${this.markdownMatchRegEx.source}(.*)`);
-        this.defaultCellMarker = codeLens ? codeLens?.defaultCellMarker : '# %%';
+        this.defaultCellMarker = defaultExpression ? defaultExpression : '# %%';
         this.defaultCellMarkerExec = this.createRegExp(`${this.defaultCellMarker}(.*)`, /# %%(.*)/);
     }
 

src/test/datascience/editor-integration/codewatcher.unit.test.ts

@@ -134,7 +134,10 @@ suite('DataScience Code Watcher Unit Tests', () => {
                     markdownExpression: '(?!x)x',
                     defaultCellMarker: '```python'
                 }
-            ]
+            ],
+            codeRegularExpression: undefined,
+            markdownRegularExpression: undefined,
+            defaultCellMarker: undefined
         });
         debugService.setup((d) => d.activeDebugSession).returns(() => undefined);
         vscodeNotebook.setup((d) => d.activeNotebookEditor).returns(() => undefined);
@@ -445,6 +448,44 @@ fourth line
         document.verifyAll();
     });
 
+    test('Make sure old settings still work', () => {
+        const fileName = Uri.file('test.py').fsPath;
+        const version = 1;
+        const inputText = `first line
+second line
+
+# <mydelimiter>
+third line
+
+# <mydelimiter>
+fourth line
+
+# <mydelimiterformarkdown>
+# fifth line`;
+
+        jupyterSettings.codeRegularExpression = '# <mydelimiter>';
+        jupyterSettings.markdownRegularExpression = '# <mydelimiterformarkdown>';
+
+        const document = createDocument(inputText, fileName, version, TypeMoq.Times.atLeastOnce(), true);
+
+        codeWatcher.setDocument(document.object);
+
+        // Verify meta data
+        expect(codeWatcher.uri?.fsPath).to.be.equal(fileName, 'File name of CodeWatcher does not match');
+        expect(codeWatcher.getVersion()).to.be.equal(version, 'File version of CodeWatcher does not match');
+
+        // Verify code lenses
+        const codeLenses = codeWatcher.getCodeLenses();
+        expect(codeLenses.length).to.be.equal(14, 'Incorrect count of code lenses');
+
+        verifyCodeLensesAtPosition(codeLenses, 0, new Range(3, 0, 5, 0), true);
+        verifyCodeLensesAtPosition(codeLenses, 6, new Range(6, 0, 8, 0));
+        verifyCodeLensesAtPosition(codeLenses, 12, new Range(9, 0, 10, 12), false, true);
+
+        // Verify function calls
+        document.verifyAll();
+    });
+
     test('Test the RunCell command', async () => {
         const fileName = Uri.file('test.py');
         const version = 1;