Merge branch 'master' into feature-add-folder-option

omnivore-app · Feb 22, 2023 · b836f41 · b836f41
2 parents 7f8b70a + 88c9ff2
commit b836f41
Show file tree

Hide file tree

Showing 7 changed files with 183 additions and 53 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,14 @@
+## [1.0.5](https://github.com/omnivore-app/obsidian-omnivore/compare/1.0.4...1.0.5) (2023-02-22)
+
+
+### Bug Fixes
+
+* add available variables to the template config description ([09f8c83](https://github.com/omnivore-app/obsidian-omnivore/commit/09f8c832cecffbc2800d22b1bcc87cb36d34910d))
+* add date_published in the frontmatter ([cf61925](https://github.com/omnivore-app/obsidian-omnivore/commit/cf61925243bb6cfccdbc625b6069d54600daf68c))
+* remove content from default template ([11a0202](https://github.com/omnivore-app/obsidian-omnivore/commit/11a020294922c7c1caad72063f6544cf2a2cb9e7))
+* use default template if empty ([7b4f7e4](https://github.com/omnivore-app/obsidian-omnivore/commit/7b4f7e4321e3b6ee42b1e635300dc8d07c290cd7))
+* use omnivore title as the filename and add omnivore id to the frontmatter which is used to deduplicate files with the same name ([8fbf85d](https://github.com/omnivore-app/obsidian-omnivore/commit/8fbf85d86baa23f065e9bf63bd8aef607609392c))
+
 ## [1.0.4](https://github.com/omnivore-app/obsidian-omnivore/compare/1.0.3...1.0.4) (2023-02-19)
 
 

diff --git a/manifest.json b/manifest.json
@@ -1,7 +1,7 @@
 {
 	"id": "obsidian-omnivore",
 	"name": "Omnivore",
-	"version": "1.0.4",
+	"version": "1.0.5",
 	"minAppVersion": "0.15.0",
 	"description": "This is an Omnivore plugin for Obsidian.",
 	"author": "Omnivore",

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "obsidian-omnivore",
-	"version": "1.0.4",
+	"version": "1.0.5",
 	"description": "This is an Omnivore plugin for Obsidian.",
 	"main": "main.js",
 	"scripts": {

diff --git a/src/main.ts b/src/main.ts
@@ -8,18 +8,20 @@ import {
   Plugin,
   PluginSettingTab,
   Setting,
+  stringifyYaml,
   TFile,
   TFolder,
 } from "obsidian";
 import {
   Article,
   compareHighlightsInFile,
   DATE_FORMAT,
+  formatDate,
   getHighlightLocation,
   loadArticles,
   PageType,
   parseDateTime,
-  unicodeSlug,
+  replaceIllegalChars,
 } from "./util";
 import { FolderSuggest } from "./settings/file-suggest";
 
@@ -57,28 +59,11 @@ const DEFAULT_SETTINGS: Settings = {
   filter: "HIGHLIGHTS",
   syncAt: "",
   customQuery: "",
-  template: `---
-title: {{{title}}}
-{{#author}}
-author: {{{author}}}
-{{/author}}
-{{#labels.length}}
-tags:
-{{#labels}}  - {{{name}}}
-{{/labels}}
-{{/labels.length}}
-date_saved: {{{dateSaved}}}
----
-
-# {{{title}}}
+  template: `# {{{title}}}
 #Omnivore
 
 [Read on Omnivore]({{{omnivoreUrl}}})
 [Read Original]({{{originalUrl}}})
-{{#content}}
-
-{{{content}}}
-{{/content}}
 
 {{#highlights.length}}
 ## Highlights
@@ -181,7 +166,7 @@ export default class OmnivorePlugin extends Plugin {
     await this.saveSettings();
 
     try {
-      console.log(`obsidian-omnivore starting sync since: '${syncAt}`);
+      console.log(`obsidian-omnivore starting sync since: '${syncAt}'`);
 
       new Notice("🚀 Fetching articles ...");
 
@@ -203,12 +188,8 @@ export default class OmnivorePlugin extends Plugin {
         );
 
         for (const article of articles) {
-          const dateSaved = DateTime.fromISO(article.savedAt).toFormat(
-            this.settings.dateSavedFormat
-          );
-          const subFolderName = DateTime.fromISO(article.savedAt).toFormat(
-            this.settings.folderFormat
-          );
+          const dateFormat = this.settings.dateSavedFormat;
+          const subFolderName = formatDate(article.savedAt, this.settings.folderFormat);
           let folderName;
           if (this.settings.useFolder) {
             folderName = `${folder}/${subFolderName}`;
@@ -222,15 +203,6 @@ export default class OmnivorePlugin extends Plugin {
             await this.app.vault.createFolder(folderName);
           }
 
-          // use unicode slug to show characters from other languages in the file name
-          const pageName = `${folderName}/${unicodeSlug(
-            article.title,
-            article.savedAt
-          )}.md`;
-          const siteName =
-            article.siteName ||
-            this.siteNameFromUrl(article.originalArticleUrl);
-
           // sort highlights by location if selected in options
           highlightOrder === "LOCATION" &&
             article.highlights?.sort((a, b) => {
@@ -248,7 +220,6 @@ export default class OmnivorePlugin extends Plugin {
                 return compareHighlightsInFile(a, b);
               }
             });
-
           const highlights = article.highlights?.map((highlight) => {
             return {
               text: highlight.quote,
@@ -259,9 +230,13 @@ export default class OmnivorePlugin extends Plugin {
               note: highlight.annotation,
             };
           });
-
+          const dateSaved = formatDate(article.savedAt, dateFormat);
+          const siteName =
+            article.siteName ||
+            this.siteNameFromUrl(article.originalArticleUrl);
           // Build content string based on template
           const content = Mustache.render(template, {
+            id: article.id,
             title: article.title,
             omnivoreUrl: `https://omnivore.app/me/${article.slug}`,
             siteName,
@@ -276,16 +251,91 @@ export default class OmnivorePlugin extends Plugin {
             highlights,
             content: article.content,
           });
-
+          const publishedAt = article.publishedAt;
+          const datePublished = publishedAt
+            ? formatDate(publishedAt, dateFormat)
+            : null;
+          // add frontmatter to the content
+          const frontmatter = {
+            id: article.id,
+            title: article.title,
+            author: article.author,
+            tags: article.labels?.map((l) => l.name),
+            date_saved: dateSaved,
+            date_published: datePublished,
+          };
+          // remove null and empty values from frontmatter
+          const filteredFrontmatter = Object.fromEntries(
+            Object.entries(frontmatter).filter(
+              ([_, value]) => value != null && value !== ""
+            )
+          );
+          const frontmatterYaml = stringifyYaml(filteredFrontmatter);
+          const frontmatterString = `---\n${frontmatterYaml}---`;
+          // Modify the contents of the note with the updated front matter
+          let updatedContent = content.replace(
+            /^(---[\s\S]*?---)/gm,
+            frontmatterString
+          );
+          // if the content doesn't have frontmatter, add it
+          if (!content.match(/^(---[\s\S]*?---)/gm)) {
+            updatedContent = `${frontmatterString}\n\n${content}`;
+          }
+          // use the title as the filename
+          const pageName = `${folderName}/${replaceIllegalChars(
+            article.title
+          )}.md`;
           const normalizedPath = normalizePath(pageName);
           const omnivoreFile = app.vault.getAbstractFileByPath(normalizedPath);
-          if (omnivoreFile instanceof TFile) {
-            const existingContent = await this.app.vault.read(omnivoreFile);
-            if (existingContent !== content) {
-              await this.app.vault.modify(omnivoreFile, content);
+          try {
+            if (omnivoreFile instanceof TFile) {
+              await app.fileManager.processFrontMatter(
+                omnivoreFile,
+                async (frontMatter) => {
+                  const id = frontMatter.id;
+                  if (id && id !== article.id) {
+                    // this article has the same name but different id
+                    const newPageName = `${folderName}/${replaceIllegalChars(
+                      article.title
+                    )}-${article.id}.md`;
+                    const newNormalizedPath = normalizePath(newPageName);
+                    const newOmnivoreFile =
+                      app.vault.getAbstractFileByPath(newNormalizedPath);
+                    if (newOmnivoreFile instanceof TFile) {
+                      // a file with the same name and id already exists, so we need to update it
+                      const existingContent = await this.app.vault.read(
+                        newOmnivoreFile
+                      );
+                      if (existingContent !== updatedContent) {
+                        await this.app.vault.modify(
+                          newOmnivoreFile,
+                          updatedContent
+                        );
+                      }
+                      return;
+                    }
+                    // a file with the same name but different id already exists, so we need to create it
+                    await this.app.vault.create(
+                      newNormalizedPath,
+                      updatedContent
+                    );
+                    return;
+                  }
+                  // a file with the same id already exists, so we might need to update it
+                  const existingContent = await this.app.vault.read(
+                    omnivoreFile
+                  );
+                  if (existingContent !== updatedContent) {
+                    await this.app.vault.modify(omnivoreFile, updatedContent);
+                  }
+                }
+              );
+            } else if (!omnivoreFile) {
+              // file doesn't exist, so we need to create it
+              await this.app.vault.create(normalizedPath, updatedContent);
             }
-          } else if (!omnivoreFile) {
-            await this.app.vault.create(normalizedPath, content);
+          } catch (e) {
+            console.error(e);
           }
         }
       }
@@ -446,7 +496,10 @@ class OmnivoreSettingTab extends PluginSettingTab {
             fragment.createEl("a", {
               text: "Reference",
               href: "https://github.com/janl/mustache.js/#templates",
-            })
+            }),
+            fragment.createEl("br"),
+            fragment.createEl("br"),
+            "Available variables: id, title, omnivoreUrl, siteName, originalUrl, author, content, dateSaved, labels.name, highlights.text, highlights.highlightUrl, highlights.note, highlights.dateHighlighted"
           );
         })
       )
@@ -455,7 +508,11 @@ class OmnivoreSettingTab extends PluginSettingTab {
           .setPlaceholder("Enter the template")
           .setValue(this.plugin.settings.template)
           .onChange(async (value) => {
-            this.plugin.settings.template = value;
+            // TODO: validate template
+            // if template is empty, use default template
+            this.plugin.settings.template = value
+              ? value
+              : DEFAULT_SETTINGS.template;
             await this.plugin.saveSettings();
           })
         text.inputEl.setAttr("rows", 10);

diff --git a/src/util.ts b/src/util.ts
@@ -53,6 +53,7 @@ export enum PageType {
 }
 
 export interface Article {
+  id: string;
   title: string;
   siteName: string;
   originalArticleUrl: string;
@@ -65,6 +66,7 @@ export interface Article {
   savedAt: string;
   pageType: PageType;
   content?: string;
+  publishedAt: string;
 }
 
 export interface Label {
@@ -119,9 +121,60 @@ export const loadArticles = async (
   const res = await requestUrl({
     url: endpoint,
     headers: requestHeaders(apiKey),
-    body: `{"query":"\\n    query Search($after: String, $first: Int, $query: String, $includeContent: Boolean, $format: String) {\\n      search(first: $first, after: $after, query: $query, includeContent: $includeContent, format: $format) {\\n        ... on SearchSuccess {\\n          edges {\\n            node {\\n              title\\n              slug\\n              siteName\\n              originalArticleUrl\\n              url\\n              author\\n              updatedAt\\n              description\\n              savedAt\\n            pageType\\n            content\\n            highlights {\\n            id\\n        quote\\n        annotation\\n        patch\\n        updatedAt\\n          }\\n        labels {\\n            name\\n          }\\n            }\\n          }\\n          pageInfo {\\n            hasNextPage\\n          }\\n        }\\n        ... on SearchError {\\n          errorCodes\\n        }\\n      }\\n    }\\n  ","variables":{"after":"${after}","first":${first}, "query":"${
-      updatedAt ? "updated:" + updatedAt : ""
-    } sort:saved-asc ${query}", "includeContent": ${includeContent}, "format": "${format}"}}`,
+    body: JSON.stringify({
+      query: `
+        query Search($after: String, $first: Int, $query: String, $includeContent: Boolean, $format: String) {
+          search(first: $first, after: $after, query: $query, includeContent: $includeContent, format: $format) {
+            ... on SearchSuccess {
+              edges {
+                node {
+                  id
+                  title
+                  slug
+                  siteName
+                  originalArticleUrl
+                  url
+                  author
+                  updatedAt
+                  description
+                  savedAt
+                  pageType
+                  content
+                  publishedAt
+                  highlights {
+                    id
+                    quote
+                    annotation
+                    patch
+                    updatedAt
+                    labels {
+                      name
+                    }
+                  }
+                  labels {
+                    name
+                  }
+                }
+              }
+              pageInfo {
+                hasNextPage
+              }
+            }
+            ... on SearchError {
+              errorCodes
+            }
+          }
+        }`,
+      variables: {
+        after: `${after}`,
+        first,
+        query: `${
+          updatedAt ? "updated:" + updatedAt : ""
+        } sort:saved-asc ${query}`,
+        includeContent,
+        format,
+      },
+    }),
     method: "POST",
   });
 
@@ -227,3 +280,11 @@ export const unicodeSlug = (str: string, savedAt: string) => {
     new Date(savedAt).getTime().toString(16)
   );
 };
+
+export const replaceIllegalChars = (str: string): string => {
+  return str.replace(/[/\\?%*:|"<>]/g, "-");
+};
+
+export const formatDate = (date: string, format: string): string => {
+  return DateTime.fromISO(date).toFormat(format);
+};
diff --git a/versions.json b/versions.json
@@ -3,5 +3,6 @@
 	"1.0.1": "0.15.0",
 	"1.0.2": "0.15.0",
 	"1.0.3": "0.15.0",
-	"1.0.4": "0.15.0"
+	"1.0.4": "0.15.0",
+	"1.0.5": "0.15.0"
 }