From 8fbf85d86baa23f065e9bf63bd8aef607609392c Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 20 Feb 2023 23:14:52 +0800 Subject: [PATCH 1/7] fix: use omnivore title as the filename and add omnivore id to the frontmatter which is used to deduplicate files with the same name --- package-lock.json | 4 +-- src/main.ts | 91 ++++++++++++++++++++++++++++++++++++++++------- src/util.ts | 7 +++- 3 files changed, 86 insertions(+), 16 deletions(-) diff --git a/package-lock.json b/package-lock.json index d518777..7a44046 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "obsidian-omnivore", - "version": "1.0.0", + "version": "1.0.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "obsidian-omnivore", - "version": "1.0.0", + "version": "1.0.4", "license": "MIT", "dependencies": { "diff-match-patch": "^1.0.5", diff --git a/src/main.ts b/src/main.ts index 2e154ae..b2fa1e2 100644 --- a/src/main.ts +++ b/src/main.ts @@ -8,6 +8,7 @@ import { Plugin, PluginSettingTab, Setting, + stringifyYaml, TFile, TFolder, } from "obsidian"; @@ -19,7 +20,7 @@ import { loadArticles, PageType, parseDateTime, - unicodeSlug, + replaceIllegalChars, } from "./util"; import { FolderSuggest } from "./settings/file-suggest"; @@ -57,6 +58,7 @@ const DEFAULT_SETTINGS: Settings = { syncAt: "", customQuery: "", template: `--- +id: {{{id}}} title: {{{title}}} {{#author}} author: {{{author}}} @@ -179,7 +181,7 @@ export default class OmnivorePlugin extends Plugin { await this.saveSettings(); try { - console.log(`obsidian-omnivore starting sync since: '${syncAt}`); + console.log(`obsidian-omnivore starting sync since: '${syncAt}'`); new Notice("🚀 Fetching articles ..."); @@ -215,11 +217,6 @@ export default class OmnivorePlugin extends Plugin { await this.app.vault.createFolder(folderName); } - // use unicode slug to show characters from other languages in the file name - const pageName = `${folderName}/${unicodeSlug( - article.title, - article.savedAt - )}.md`; const siteName = article.siteName || this.siteNameFromUrl(article.originalArticleUrl); @@ -255,6 +252,7 @@ export default class OmnivorePlugin extends Plugin { // Build content string based on template const content = Mustache.render(template, { + id: article.id, title: article.title, omnivoreUrl: `https://omnivore.app/me/${article.slug}`, siteName, @@ -270,15 +268,82 @@ export default class OmnivorePlugin extends Plugin { content: article.content, }); + // add frontmatter to the content + const frontmatter = { + id: article.id, + title: article.title, + author: article.author, + tags: article.labels?.map((l) => l.name), + date_saved: dateSaved, + }; + // remove null and empty values from frontmatter + const filteredFrontmatter = Object.fromEntries( + Object.entries(frontmatter).filter( + ([_, value]) => value != null && value !== "" + ) + ); + const frontmatterYaml = stringifyYaml(filteredFrontmatter); + const frontmatterString = `---\n${frontmatterYaml}---`; + // Modify the contents of the note with the updated front matter + const updatedContent = content.replace( + /^(---[\s\S]*?---)/gm, + frontmatterString + ); + // use the title as the filename + const pageName = `${folderName}/${replaceIllegalChars( + article.title + )}.md`; const normalizedPath = normalizePath(pageName); const omnivoreFile = app.vault.getAbstractFileByPath(normalizedPath); - if (omnivoreFile instanceof TFile) { - const existingContent = await this.app.vault.read(omnivoreFile); - if (existingContent !== content) { - await this.app.vault.modify(omnivoreFile, content); + try { + if (omnivoreFile instanceof TFile) { + await app.fileManager.processFrontMatter( + omnivoreFile, + async (frontMatter) => { + const id = frontMatter.id; + if (id && id !== article.id) { + // this article has the same name but different id + const newPageName = `${folderName}/${replaceIllegalChars( + article.title + )}-${article.id}.md`; + const newNormalizedPath = normalizePath(newPageName); + const newOmnivoreFile = + app.vault.getAbstractFileByPath(newNormalizedPath); + if (newOmnivoreFile instanceof TFile) { + // a file with the same name and id already exists, so we need to update it + const existingContent = await this.app.vault.read( + newOmnivoreFile + ); + if (existingContent !== updatedContent) { + await this.app.vault.modify( + newOmnivoreFile, + updatedContent + ); + } + return; + } + // a file with the same name but different id already exists, so we need to create it + await this.app.vault.create( + newNormalizedPath, + updatedContent + ); + return; + } + // a file with the same id already exists, so we might need to update it + const existingContent = await this.app.vault.read( + omnivoreFile + ); + if (existingContent !== updatedContent) { + await this.app.vault.modify(omnivoreFile, updatedContent); + } + } + ); + } else if (!omnivoreFile) { + // file doesn't exist, so we need to create it + await this.app.vault.create(normalizedPath, updatedContent); } - } else if (!omnivoreFile) { - await this.app.vault.create(normalizedPath, content); + } catch (e) { + console.error(e); } } } diff --git a/src/util.ts b/src/util.ts index cec47aa..55cb307 100644 --- a/src/util.ts +++ b/src/util.ts @@ -53,6 +53,7 @@ export enum PageType { } export interface Article { + id: string; title: string; siteName: string; originalArticleUrl: string; @@ -119,7 +120,7 @@ export const loadArticles = async ( const res = await requestUrl({ url: endpoint, headers: requestHeaders(apiKey), - body: `{"query":"\\n query Search($after: String, $first: Int, $query: String, $includeContent: Boolean, $format: String) {\\n search(first: $first, after: $after, query: $query, includeContent: $includeContent, format: $format) {\\n ... on SearchSuccess {\\n edges {\\n node {\\n title\\n slug\\n siteName\\n originalArticleUrl\\n url\\n author\\n updatedAt\\n description\\n savedAt\\n pageType\\n content\\n highlights {\\n id\\n quote\\n annotation\\n patch\\n updatedAt\\n }\\n labels {\\n name\\n }\\n }\\n }\\n pageInfo {\\n hasNextPage\\n }\\n }\\n ... on SearchError {\\n errorCodes\\n }\\n }\\n }\\n ","variables":{"after":"${after}","first":${first}, "query":"${ + body: `{"query":"\\n query Search($after: String, $first: Int, $query: String, $includeContent: Boolean, $format: String) {\\n search(first: $first, after: $after, query: $query, includeContent: $includeContent, format: $format) {\\n ... on SearchSuccess {\\n edges {\\n node {\\n title\\n slug\\n siteName\\n originalArticleUrl\\n url\\n author\\n updatedAt\\n description\\n savedAt\\n pageType\\n content\\n id\\n highlights {\\n id\\n quote\\n annotation\\n patch\\n updatedAt\\n }\\n labels {\\n name\\n }\\n }\\n }\\n pageInfo {\\n hasNextPage\\n }\\n }\\n ... on SearchError {\\n errorCodes\\n }\\n }\\n }\\n ","variables":{"after":"${after}","first":${first}, "query":"${ updatedAt ? "updated:" + updatedAt : "" } sort:saved-asc ${query}", "includeContent": ${includeContent}, "format": "${format}"}}`, method: "POST", @@ -227,3 +228,7 @@ export const unicodeSlug = (str: string, savedAt: string) => { new Date(savedAt).getTime().toString(16) ); }; + +export const replaceIllegalChars = (str: string): string => { + return str.replace(/[/\\?%*:|"<>]/g, "-"); +}; From 4dfcadfd70f58bb7449f6db17259f9019d28f8ab Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 21 Feb 2023 09:48:10 +0800 Subject: [PATCH 2/7] prepend frontmatter if not exist --- src/main.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main.ts b/src/main.ts index b2fa1e2..a81cda9 100644 --- a/src/main.ts +++ b/src/main.ts @@ -217,10 +217,6 @@ export default class OmnivorePlugin extends Plugin { await this.app.vault.createFolder(folderName); } - const siteName = - article.siteName || - this.siteNameFromUrl(article.originalArticleUrl); - // sort highlights by location if selected in options highlightOrder === "LOCATION" && article.highlights?.sort((a, b) => { @@ -238,7 +234,6 @@ export default class OmnivorePlugin extends Plugin { return compareHighlightsInFile(a, b); } }); - const highlights = article.highlights?.map((highlight) => { return { text: highlight.quote, @@ -249,7 +244,9 @@ export default class OmnivorePlugin extends Plugin { note: highlight.annotation, }; }); - + const siteName = + article.siteName || + this.siteNameFromUrl(article.originalArticleUrl); // Build content string based on template const content = Mustache.render(template, { id: article.id, @@ -267,7 +264,6 @@ export default class OmnivorePlugin extends Plugin { highlights, content: article.content, }); - // add frontmatter to the content const frontmatter = { id: article.id, @@ -285,10 +281,14 @@ export default class OmnivorePlugin extends Plugin { const frontmatterYaml = stringifyYaml(filteredFrontmatter); const frontmatterString = `---\n${frontmatterYaml}---`; // Modify the contents of the note with the updated front matter - const updatedContent = content.replace( + let updatedContent = content.replace( /^(---[\s\S]*?---)/gm, frontmatterString ); + // if the content doesn't have frontmatter, add it + if (!content.match(/^(---[\s\S]*?---)/gm)) { + updatedContent = `${frontmatterString}\n\n${content}`; + } // use the title as the filename const pageName = `${folderName}/${replaceIllegalChars( article.title From 09f8c832cecffbc2800d22b1bcc87cb36d34910d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 21 Feb 2023 10:03:11 +0800 Subject: [PATCH 3/7] fix: add available variables to the template config description --- src/main.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main.ts b/src/main.ts index a81cda9..3f74c83 100644 --- a/src/main.ts +++ b/src/main.ts @@ -504,7 +504,9 @@ class OmnivoreSettingTab extends PluginSettingTab { fragment.createEl("a", { text: "Reference", href: "https://github.com/janl/mustache.js/#templates", - }) + }), + fragment.createEl("br"), + "Available variables: id, title, omnivoreUrl, siteName, originalUrl, author, content, dateSaved, labels.name, highlights.text, highlights.highlightUrl, highlights.note, highlights.dateHighlighted" ); }) ) From 11a020294922c7c1caad72063f6544cf2a2cb9e7 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 21 Feb 2023 20:23:32 +0800 Subject: [PATCH 4/7] fix: remove content from default template --- src/main.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main.ts b/src/main.ts index 3f74c83..0bb3375 100644 --- a/src/main.ts +++ b/src/main.ts @@ -76,10 +76,6 @@ date_saved: {{{dateSaved}}} [Read on Omnivore]({{{omnivoreUrl}}}) [Read Original]({{{originalUrl}}}) -{{#content}} - -{{{content}}} -{{/content}} {{#highlights.length}} ## Highlights From 7b4f7e4321e3b6ee42b1e635300dc8d07c290cd7 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 22 Feb 2023 16:01:13 +0800 Subject: [PATCH 5/7] fix: use default template if empty --- src/main.ts | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/src/main.ts b/src/main.ts index 0bb3375..7357a90 100644 --- a/src/main.ts +++ b/src/main.ts @@ -57,21 +57,7 @@ const DEFAULT_SETTINGS: Settings = { filter: "HIGHLIGHTS", syncAt: "", customQuery: "", - template: `--- -id: {{{id}}} -title: {{{title}}} -{{#author}} -author: {{{author}}} -{{/author}} -{{#labels.length}} -tags: -{{#labels}} - {{{name}}} -{{/labels}} -{{/labels.length}} -date_saved: {{{dateSaved}}} ---- - -# {{{title}}} + template: `# {{{title}}} #Omnivore [Read on Omnivore]({{{omnivoreUrl}}}) @@ -502,6 +488,7 @@ class OmnivoreSettingTab extends PluginSettingTab { href: "https://github.com/janl/mustache.js/#templates", }), fragment.createEl("br"), + fragment.createEl("br"), "Available variables: id, title, omnivoreUrl, siteName, originalUrl, author, content, dateSaved, labels.name, highlights.text, highlights.highlightUrl, highlights.note, highlights.dateHighlighted" ); }) @@ -511,7 +498,11 @@ class OmnivoreSettingTab extends PluginSettingTab { .setPlaceholder("Enter the template") .setValue(this.plugin.settings.template) .onChange(async (value) => { - this.plugin.settings.template = value; + // TODO: validate template + // if template is empty, use default template + this.plugin.settings.template = value + ? value + : DEFAULT_SETTINGS.template; await this.plugin.saveSettings(); }) ); From cf61925243bb6cfccdbc625b6069d54600daf68c Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 22 Feb 2023 16:01:36 +0800 Subject: [PATCH 6/7] fix: add date_published in the frontmatter --- src/main.ts | 15 +++++++------ src/util.ts | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 68 insertions(+), 9 deletions(-) diff --git a/src/main.ts b/src/main.ts index 7357a90..1f01bbe 100644 --- a/src/main.ts +++ b/src/main.ts @@ -16,6 +16,7 @@ import { Article, compareHighlightsInFile, DATE_FORMAT, + formatDate, getHighlightLocation, loadArticles, PageType, @@ -185,12 +186,8 @@ export default class OmnivorePlugin extends Plugin { ); for (const article of articles) { - const dateSaved = DateTime.fromISO(article.savedAt).toFormat( - this.settings.dateSavedFormat - ); - const subFolderName = DateTime.fromISO(article.savedAt).toFormat( - this.settings.folderFormat - ); + const dateFormat = this.settings.dateSavedFormat; + const subFolderName = formatDate(article.savedAt, this.settings.folderFormat); const folderName = `${folder}/${subFolderName}`; const omnivoreFolder = app.vault.getAbstractFileByPath( normalizePath(folderName) @@ -226,6 +223,7 @@ export default class OmnivorePlugin extends Plugin { note: highlight.annotation, }; }); + const dateSaved = formatDate(article.savedAt, dateFormat); const siteName = article.siteName || this.siteNameFromUrl(article.originalArticleUrl); @@ -246,6 +244,10 @@ export default class OmnivorePlugin extends Plugin { highlights, content: article.content, }); + const publishedAt = article.publishedAt; + const datePublished = publishedAt + ? formatDate(publishedAt, dateFormat) + : null; // add frontmatter to the content const frontmatter = { id: article.id, @@ -253,6 +255,7 @@ export default class OmnivorePlugin extends Plugin { author: article.author, tags: article.labels?.map((l) => l.name), date_saved: dateSaved, + date_published: datePublished, }; // remove null and empty values from frontmatter const filteredFrontmatter = Object.fromEntries( diff --git a/src/util.ts b/src/util.ts index 55cb307..24d7991 100644 --- a/src/util.ts +++ b/src/util.ts @@ -66,6 +66,7 @@ export interface Article { savedAt: string; pageType: PageType; content?: string; + publishedAt: string; } export interface Label { @@ -120,9 +121,60 @@ export const loadArticles = async ( const res = await requestUrl({ url: endpoint, headers: requestHeaders(apiKey), - body: `{"query":"\\n query Search($after: String, $first: Int, $query: String, $includeContent: Boolean, $format: String) {\\n search(first: $first, after: $after, query: $query, includeContent: $includeContent, format: $format) {\\n ... on SearchSuccess {\\n edges {\\n node {\\n title\\n slug\\n siteName\\n originalArticleUrl\\n url\\n author\\n updatedAt\\n description\\n savedAt\\n pageType\\n content\\n id\\n highlights {\\n id\\n quote\\n annotation\\n patch\\n updatedAt\\n }\\n labels {\\n name\\n }\\n }\\n }\\n pageInfo {\\n hasNextPage\\n }\\n }\\n ... on SearchError {\\n errorCodes\\n }\\n }\\n }\\n ","variables":{"after":"${after}","first":${first}, "query":"${ - updatedAt ? "updated:" + updatedAt : "" - } sort:saved-asc ${query}", "includeContent": ${includeContent}, "format": "${format}"}}`, + body: JSON.stringify({ + query: ` + query Search($after: String, $first: Int, $query: String, $includeContent: Boolean, $format: String) { + search(first: $first, after: $after, query: $query, includeContent: $includeContent, format: $format) { + ... on SearchSuccess { + edges { + node { + id + title + slug + siteName + originalArticleUrl + url + author + updatedAt + description + savedAt + pageType + content + publishedAt + highlights { + id + quote + annotation + patch + updatedAt + labels { + name + } + } + labels { + name + } + } + } + pageInfo { + hasNextPage + } + } + ... on SearchError { + errorCodes + } + } + }`, + variables: { + after: `${after}`, + first, + query: `${ + updatedAt ? "updated:" + updatedAt : "" + } sort:saved-asc ${query}`, + includeContent, + format, + }, + }), method: "POST", }); @@ -232,3 +284,7 @@ export const unicodeSlug = (str: string, savedAt: string) => { export const replaceIllegalChars = (str: string): string => { return str.replace(/[/\\?%*:|"<>]/g, "-"); }; + +export const formatDate = (date: string, format: string): string => { + return DateTime.fromISO(date).toFormat(format); +}; From 88c9ff2710a4df72b90b9e1c6c9f5d1f8fa7cbec Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 22 Feb 2023 08:10:58 +0000 Subject: [PATCH 7/7] chore(release): 1.0.5 ## [1.0.5](https://github.com/omnivore-app/obsidian-omnivore/compare/1.0.4...1.0.5) (2023-02-22) ### Bug Fixes * add available variables to the template config description ([09f8c83](https://github.com/omnivore-app/obsidian-omnivore/commit/09f8c832cecffbc2800d22b1bcc87cb36d34910d)) * add date_published in the frontmatter ([cf61925](https://github.com/omnivore-app/obsidian-omnivore/commit/cf61925243bb6cfccdbc625b6069d54600daf68c)) * remove content from default template ([11a0202](https://github.com/omnivore-app/obsidian-omnivore/commit/11a020294922c7c1caad72063f6544cf2a2cb9e7)) * use default template if empty ([7b4f7e4](https://github.com/omnivore-app/obsidian-omnivore/commit/7b4f7e4321e3b6ee42b1e635300dc8d07c290cd7)) * use omnivore title as the filename and add omnivore id to the frontmatter which is used to deduplicate files with the same name ([8fbf85d](https://github.com/omnivore-app/obsidian-omnivore/commit/8fbf85d86baa23f065e9bf63bd8aef607609392c)) --- CHANGELOG.md | 11 +++++++++++ manifest.json | 2 +- package.json | 2 +- versions.json | 3 ++- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c68262f..9d5a42c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +## [1.0.5](https://github.com/omnivore-app/obsidian-omnivore/compare/1.0.4...1.0.5) (2023-02-22) + + +### Bug Fixes + +* add available variables to the template config description ([09f8c83](https://github.com/omnivore-app/obsidian-omnivore/commit/09f8c832cecffbc2800d22b1bcc87cb36d34910d)) +* add date_published in the frontmatter ([cf61925](https://github.com/omnivore-app/obsidian-omnivore/commit/cf61925243bb6cfccdbc625b6069d54600daf68c)) +* remove content from default template ([11a0202](https://github.com/omnivore-app/obsidian-omnivore/commit/11a020294922c7c1caad72063f6544cf2a2cb9e7)) +* use default template if empty ([7b4f7e4](https://github.com/omnivore-app/obsidian-omnivore/commit/7b4f7e4321e3b6ee42b1e635300dc8d07c290cd7)) +* use omnivore title as the filename and add omnivore id to the frontmatter which is used to deduplicate files with the same name ([8fbf85d](https://github.com/omnivore-app/obsidian-omnivore/commit/8fbf85d86baa23f065e9bf63bd8aef607609392c)) + ## [1.0.4](https://github.com/omnivore-app/obsidian-omnivore/compare/1.0.3...1.0.4) (2023-02-19) diff --git a/manifest.json b/manifest.json index bfd6dd3..2f9718d 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "obsidian-omnivore", "name": "Omnivore", - "version": "1.0.4", + "version": "1.0.5", "minAppVersion": "0.15.0", "description": "This is an Omnivore plugin for Obsidian.", "author": "Omnivore", diff --git a/package.json b/package.json index d6dd4f8..21338e6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "obsidian-omnivore", - "version": "1.0.4", + "version": "1.0.5", "description": "This is an Omnivore plugin for Obsidian.", "main": "main.js", "scripts": { diff --git a/versions.json b/versions.json index 1d7eb49..22b4387 100644 --- a/versions.json +++ b/versions.json @@ -3,5 +3,6 @@ "1.0.1": "0.15.0", "1.0.2": "0.15.0", "1.0.3": "0.15.0", - "1.0.4": "0.15.0" + "1.0.4": "0.15.0", + "1.0.5": "0.15.0" } \ No newline at end of file