From b24aa8c46f837da99c91a901aee290634fa61a1f Mon Sep 17 00:00:00 2001 From: Jeff Daley Date: Fri, 3 Mar 2023 15:07:50 -0500 Subject: [PATCH] Fixes errors caused by special characters (#17) * Add `cleanString` function * Update yarn.lock --- web/app/components/document/sidebar.js | 8 ++- web/app/components/new/doc-form.js | 13 ++-- web/app/utils/clean-string.ts | 74 +++++++++++++++++++++++ web/package.json | 1 + web/tests/unit/utils/clean-string-test.ts | 40 ++++++++++++ web/yarn.lock | 8 +++ 6 files changed, 137 insertions(+), 7 deletions(-) create mode 100644 web/app/utils/clean-string.ts create mode 100644 web/tests/unit/utils/clean-string-test.ts diff --git a/web/app/components/document/sidebar.js b/web/app/components/document/sidebar.js index 1127b2d0d..9bdaf368c 100644 --- a/web/app/components/document/sidebar.js +++ b/web/app/components/document/sidebar.js @@ -5,6 +5,7 @@ import { getOwner } from "@ember/application"; import { inject as service } from "@ember/service"; import { task } from "ember-concurrency"; import { dasherize } from "@ember/string"; +import cleanString from "hermes/utils/clean-string"; export default class DocumentSidebar extends Component { @service("fetch") fetchSvc; @@ -212,13 +213,16 @@ export default class DocumentSidebar extends Component { *save(field, val) { if (field && val) { const oldVal = this[field]; - this[field] = val; + this[field] = cleanString(val); try { const serializedValue = this.emailFields.includes(field) ? val.map((p) => p.email) : val; - yield this.patchDocument.perform({ [field]: serializedValue }); + + yield this.patchDocument.perform({ + [field]: cleanString(serializedValue), + }); } catch (err) { // revert field value on failure this[field] = oldVal; diff --git a/web/app/components/new/doc-form.js b/web/app/components/new/doc-form.js index a98c00e07..f8a6a8d4e 100644 --- a/web/app/components/new/doc-form.js +++ b/web/app/components/new/doc-form.js @@ -3,7 +3,8 @@ import { task } from "ember-concurrency"; import { inject as service } from "@ember/service"; import { tracked } from "@glimmer/tracking"; import { action } from "@ember/object"; -import Ember from 'ember'; +import Ember from "ember"; +import cleanString from "hermes/utils/clean-string"; const FORM_ERRORS = { title: null, @@ -144,8 +145,8 @@ export default class NewDocForm extends Component { owner: this.authenticatedUser.info.email, product: this.productArea, productAbbreviation: this.productAbbreviation, - summary: this.summary, - title: this.title, + summary: cleanString(this.summary), + title: cleanString(this.title), tags: this.tags, }), }) @@ -160,12 +161,14 @@ export default class NewDocForm extends Component { yield wait(AWAIT_DOC_DELAY); // Set modal on a delay so it appears after transition. - this.modalAlerts.setActive.perform("docCreated", AWAIT_DOC_CREATED_MODAL_DELAY); + this.modalAlerts.setActive.perform( + "docCreated", + AWAIT_DOC_CREATED_MODAL_DELAY + ); this.router.transitionTo("authenticated.document", doc.id, { queryParams: { draft: true }, }); - } catch (err) { this.docIsBeingCreated = false; // TODO: Handle error by using a toast and showing the create form again with diff --git a/web/app/utils/clean-string.ts b/web/app/utils/clean-string.ts new file mode 100644 index 000000000..9443b071f --- /dev/null +++ b/web/app/utils/clean-string.ts @@ -0,0 +1,74 @@ +import replaceSpecialCharacters from "replace-special-characters"; + +export default function cleanString(string: string): string { + // ‘, ’, ‚ + let newString = string.replace(/[\u2018\u2019\u201A]/g, "'"); + // “, ”, „ + newString = newString.replace(/[\u201C\u201D\u201E]/g, '"'); + // … + newString = newString.replace(/\u2026/g, "..."); + // – + newString = newString.replace(/\u2013/g, "-"); + // — + newString = newString.replace(/\u2014/g, "--"); + // •, · + newString = newString.replace(/[\u2022\u00B7]/g, "*"); + // « + newString = newString.replace(/\u00AB/g, "<<"); + // » + newString = newString.replace(/\u00BB/g, ">>"); + // ‹ + newString = newString.replace(/\u2039/g, "<"); + // › + newString = newString.replace(/\u203A/g, ">"); + // ¢ + newString = newString.replace(/\u00A2/g, "c"); + // © + newString = newString.replace(/\u00A9/g, "(C)"); + // ™ + newString = newString.replace(/\u2122/g, "(TM)"); + // ® + newString = newString.replace(/\u00AE/g, "(R)"); + // ° + newString = newString.replace(/\u00B0/g, "(deg)"); + // ± + newString = newString.replace(/\u00B1/g, "+/-"); + // × + newString = newString.replace(/\u00D7/g, "*"); + // ÷ + newString = newString.replace(/\u00F7/g, "/"); + // ¼ + newString = newString.replace(/\u00BC/g, "1/4"); + // ½ + newString = newString.replace(/\u00BD/g, "1/2"); + // ¾ + newString = newString.replace(/\u00BE/g, "3/4"); + // √ + newString = newString.replace(/\u221A/g, "sqrt"); + // ¹ + newString = newString.replace(/\u00B9/g, "^1"); + // ² + newString = newString.replace(/\u00B2/g, "^2"); + // ³ + newString = newString.replace(/\u00B3/g, "^3"); + // ⁿ + newString = newString.replace(/\u207F/g, "^n"); + // ¿ + newString = newString.replace(/\u00BF/g, "?"); + // ¡ + newString = newString.replace(/\u00A1/g, "!"); + // ≥ + newString = newString.replace(/\u2265/g, ">="); + // ≤ + newString = newString.replace(/\u2264/g, "<="); + // ≠ + newString = newString.replace(/\u2260/g, "!="); + + // replace special characters e.g., é with e + newString = replaceSpecialCharacters(newString); + + // remove non-ascii characters + newString = newString.replace(/[^\x00-\x7F]/g, ""); + + return newString; +} diff --git a/web/package.json b/web/package.json index 7fd2de912..77be76416 100644 --- a/web/package.json +++ b/web/package.json @@ -128,6 +128,7 @@ "instantsearch.js": "^4.40.2", "miragejs": "^0.1.47", "postcss-scss": "^4.0.3", + "replace-special-characters": "^1.2.7", "tailwindcss": "^3.0.23", "torii": "^0.10.1", "webpack": "^5.70.0" diff --git a/web/tests/unit/utils/clean-string-test.ts b/web/tests/unit/utils/clean-string-test.ts new file mode 100644 index 000000000..8da9269af --- /dev/null +++ b/web/tests/unit/utils/clean-string-test.ts @@ -0,0 +1,40 @@ +import { module, test } from "qunit"; +import cleanString from "hermes/utils/clean-string"; + +module("Unit | Utility | clean-string", function () { + test("it correctly cleans strings", function (assert) { + assert.equal(cleanString("\u2018\u2019\u201A"), "'''"); + assert.equal(cleanString("\u201C\u201D\u201E"), '"""'); + assert.equal(cleanString("\u2026"), "..."); + assert.equal(cleanString("\u2013"), "-"); + assert.equal(cleanString("\u2014"), "--"); + assert.equal(cleanString("\u2022\u00B7"), "**"); + assert.equal(cleanString("\u00AB"), "<<"); + assert.equal(cleanString("\u00BB"), ">>"); + assert.equal(cleanString("\u2039"), "<"); + assert.equal(cleanString("\u203A"), ">"); + assert.equal(cleanString("\u00A2"), "c"); + assert.equal(cleanString("\u00A9"), "(C)"); + assert.equal(cleanString("\u2122"), "(TM)"); + assert.equal(cleanString("\u00AE"), "(R)"); + assert.equal(cleanString("\u00B0"), "(deg)"); + assert.equal(cleanString("\u00B1"), "+/-"); + assert.equal(cleanString("\u00D7"), "*"); + assert.equal(cleanString("\u00F7"), "/"); + assert.equal(cleanString("\u00BC"), "1/4"); + assert.equal(cleanString("\u00BD"), "1/2"); + assert.equal(cleanString("\u00BE"), "3/4"); + assert.equal(cleanString("\u221A"), "sqrt"); + assert.equal(cleanString("\u00B9"), "^1"); + assert.equal(cleanString("\u00B2"), "^2"); + assert.equal(cleanString("\u00B3"), "^3"); + assert.equal(cleanString("\u207F"), "^n"); + assert.equal(cleanString("\u00BF"), "?"); + assert.equal(cleanString("\u00A1"), "!"); + assert.equal(cleanString("\u2265"), ">="); + assert.equal(cleanString("\u2264"), "<="); + assert.equal(cleanString("\u2260"), "!="); + assert.equal(cleanString("pøkémöñ"), "pokemon"); + assert.equal(cleanString("∑π¬µ∫≈π†ºª¶§∞£"), ""); + }); +}); diff --git a/web/yarn.lock b/web/yarn.lock index 0b34a3046..5961166da 100644 --- a/web/yarn.lock +++ b/web/yarn.lock @@ -10838,6 +10838,7 @@ __metadata: prettier: ^2.5.1 qunit: ^2.17.2 qunit-dom: ^1.6.0 + replace-special-characters: ^1.2.7 tailwindcss: ^3.0.23 torii: ^0.10.1 typescript: latest @@ -15022,6 +15023,13 @@ __metadata: languageName: node linkType: hard +"replace-special-characters@npm:^1.2.7": + version: 1.2.7 + resolution: "replace-special-characters@npm:1.2.7" + checksum: cc7db80e2f63e27255cf9b3ef89c82260bb89335634f53fe8bfcd1a10bfc7603c583214dc105c05b7a6f9b97a2789ace16222664d160662ef332a02074b4c9f7 + languageName: node + linkType: hard + "require-directory@npm:^2.1.1": version: 2.1.1 resolution: "require-directory@npm:2.1.1"