diff --git a/package-lock.json b/package-lock.json index cf71161..f1b83ec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "obsidian-omnivore", - "version": "1.3.2", + "version": "1.4.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "obsidian-omnivore", - "version": "1.3.2", + "version": "1.4.2", "license": "MIT", "dependencies": { "diff-match-patch": "^1.0.5", @@ -14,6 +14,7 @@ "luxon": "^3.1.1", "markdown-escape": "^2.0.0", "mustache": "^4.2.0", + "out-of-character": "^1.2.1", "process": "^0.11.10" }, "devDependencies": { @@ -2313,8 +2314,7 @@ "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, "node_modules/before-after-hook": { "version": "2.2.3", @@ -2332,7 +2332,6 @@ "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -2596,6 +2595,11 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true }, + "node_modules/colorette": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.2.2.tgz", + "integrity": "sha512-MKGMzyfeuutC/ZJ1cba9NqcNpfeqMUcYmyF1ZFY6/Cn7CNSAKx6a+s48sqLqyAiZuaP2TcqMhoo+dlwFnVxT9w==" + }, "node_modules/compare-func": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/compare-func/-/compare-func-2.0.0.tgz", @@ -2609,8 +2613,7 @@ "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" }, "node_modules/config-chain": { "version": "1.1.13", @@ -3865,8 +3868,7 @@ "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, "node_modules/fsevents": { "version": "2.3.2", @@ -4240,7 +4242,6 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "dev": true, "dependencies": { "once": "^1.3.0", "wrappy": "1" @@ -4249,8 +4250,7 @@ "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "node_modules/ini": { "version": "1.3.8", @@ -5652,7 +5652,6 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, "dependencies": { "brace-expansion": "^1.1.7" }, @@ -8406,7 +8405,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, "dependencies": { "wrappy": "1" } @@ -8444,6 +8442,40 @@ "node": ">= 0.8.0" } }, + "node_modules/out-of-character": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/out-of-character/-/out-of-character-1.2.1.tgz", + "integrity": "sha512-clqCX4lNQa5Qr7urMNej2DbKVt+SO5CZ9Sq4XYEJ04qzvDHA4yFMhz0F+vLrfxO89640x8W56D7wtNoAgCCcAA==", + "dependencies": { + "colorette": "1.2.2", + "glob": "7.1.7" + }, + "bin": { + "out-of-character": "bin/out-of-character.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/out-of-character/node_modules/glob": { + "version": "7.1.7", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz", + "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/p-each-series": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-each-series/-/p-each-series-2.2.0.tgz", @@ -8591,7 +8623,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -10269,8 +10300,7 @@ "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "node_modules/write-file-atomic": { "version": "4.0.2", diff --git a/package.json b/package.json index 0197e6c..cf3eea7 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "luxon": "^3.1.1", "markdown-escape": "^2.0.0", "mustache": "^4.2.0", + "out-of-character": "^1.2.1", "process": "^0.11.10" }, "volta": { diff --git a/src/__tests__/path_validation.spec.ts b/src/__tests__/path_validation.spec.ts index a0f5a55..928d620 100644 --- a/src/__tests__/path_validation.spec.ts +++ b/src/__tests__/path_validation.spec.ts @@ -19,6 +19,8 @@ const expectedManualIllegalChars: string[] = [ "\u001F", ]; +const expectedInvisibleChars: string[] = ["­", "‍"]; + describe("replaceIllegalChars() removes all expected characters", () => { test.each(expectedManualIllegalChars)( 'Illegal character "%s" is removed', @@ -97,3 +99,16 @@ describe("file system behavior with non-alphanumeric characters not in the illeg } ); }); + +describe("replaceIllegalChars() function removes all occurrences of invisible characters", () => { + test.each(expectedInvisibleChars)( + "Invisible character '%s' is replaced", + (char) => { + const input = `${char}foo${char}bar`; + const expectedOutput = "foobar"; + const output = replaceIllegalChars(input); + expect(output).toEqual(expectedOutput); + expect(output.match(ILLEGAL_CHAR_REGEX)).toBeNull(); + } + ); +}); diff --git a/src/util.ts b/src/util.ts index 84398ae..ae6b0c5 100644 --- a/src/util.ts +++ b/src/util.ts @@ -2,6 +2,7 @@ import { diff_match_patch } from "diff-match-patch"; import { DateTime } from "luxon"; import escape from "markdown-escape"; import { parseYaml } from "obsidian"; +import { replace } from "out-of-character"; import { Highlight } from "./api"; export const DATE_FORMAT_W_OUT_SECONDS = "yyyy-MM-dd'T'HH:mm"; @@ -91,7 +92,8 @@ export const unicodeSlug = (str: string, savedAt: string) => { }; export const replaceIllegalChars = (str: string): string => { - return str.replace(ILLEGAL_CHAR_REGEX, REPLACEMENT_CHAR); + // remove invisible characters + return replace(str.replace(ILLEGAL_CHAR_REGEX, REPLACEMENT_CHAR)); }; export function formatDate(date: string, format: string): string {