-
Notifications
You must be signed in to change notification settings - Fork 9
/
Copy pathdenote-explore.el
executable file
·1432 lines (1293 loc) · 58.5 KB
/
denote-explore.el
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
;;; denote-explore.el --- Explore and visualise Denote files -*- lexical-binding: t -*-
;;
;; Copyright (C) 2023-2025 Peter Prevos
;;
;; Author: Peter Prevos <[email protected]>
;; URL: https://github.com/pprevos/denote-explore/
;; Version: 3.3.1
;; Package-Requires: ((emacs "29.1") (denote "3.1") (dash "2.19.1"))
;;
;; This file is NOT part of GNU Emacs.
;;
;; This program is free software; you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation, either version 3 of the License, or
;; (at your option) any later version.
;;
;; This program is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;; GNU General Public License for more details.
;;
;; You should have received a copy of the GNU General Public License
;; along with this program. If not, see <https://www.gnu.org/licenses/>.
;;
;;; Commentary:
;;
;; Denote-Explore provides functionality to explore, maintain and visualise
;; your collection of Denote files. The major version number indicates
;; compatability with the relevant Denote major version.
;;
;; Functionality:
;;
;; 1. Statistics: count and visualise notes and keywords
;; 2. Random walks: aces notes with serendipitous discovery
;; 3. Janitor: Maintenance on you Denote collection
;; 4. Network diagrams: visualise the structure of your notes
;;
;; The Denote-Explore manual is available in info-mode
;; (info "denote-explore") `C-h R denote-explore`
;;
;;; Code:
(require 'denote)
(require 'dash)
(require 'chart)
(require 'cl-lib)
(require 'json)
(require 'browse-url)
;;; CUSTOMISATION
(defgroup denote-explore ()
"Explore and visualise Denote file collections."
:group 'files
:link '(url-link :tag "Homepage" "https://github.com/pprevos/denote-explore"))
(defcustom denote-explore-network-directory
(expand-file-name "graphs/" denote-directory)
"Directory to store Denote network files.
Created upon generating the network when it does not yet exist"
:group 'denote-explore
:package-version '(denote-explore . "1.3")
:type 'string)
(define-obsolete-variable-alias
'denote-explore-json-vertices-filename
'denote-explore-network-filename "1.3")
(define-obsolete-variable-alias
'denote-explore-json-edges-filename
'denote-explore-network-filename "1.3")
(defcustom denote-explore-network-filename
"denote-network"
"Base filename sans extension for Denote explore network files.
Stored in `denote-explore-network-directory'.
File type defined by `denote-explore-network-format'."
:group 'denote-explore
:package-version '(denote-explore . "1.3")
:type 'string)
(defcustom denote-explore-network-format
'd3.js
"Output format for Denote network files."
:group 'denote-explore
:package-version '(denote-explore . "1.3")
:type '(choice
(const :tag "D3 JavaScript (JSON)" d3.js)
(const :tag "GraphViz (Dot)" graphviz)
(const :tag "Graph Exchange XML Format (GEXF)" gexf)))
(defcustom denote-explore-network-keywords-ignore '()
"List of keywords to be ignored in the keywords graph."
:group 'denote-explore
:package-version '(denote-explore . "1.3")
:type '(repeat (string :tag "Keyword")))
(defcustom denote-explore-network-regex-ignore '()
"Regular expression ignored in neighbourhood, community and sequence graphs."
:group 'denote-explore
:package-version '(denote-explore . "1.4")
:type '(choice (const :tag "No Ignore Regexp" nil)
(regexp :tag "Ignore using Regexp")))
(defcustom denote-explore-random-regex-ignore '()
"Regular expression to exclude form random walks."
:group 'denote-explore
:package-version '(denote-explore . "3.3.1")
:type '(choice (const :tag "No Ignored Regexp" nil)
(regexp :tag "Ignore using Regexp")))
(defcustom denote-explore-network-d3-template
nil
"Fully qualified path of the D3.JS HTML template file."
:group 'denote-explore
:package-version '(denote-explore . "3.1")
:type '(file :must-match t)
:initialize 'custom-initialize-default)
;; Set default value at load time if not customised
(when (not denote-explore-network-d3-template)
(setq denote-explore-network-d3-template
(expand-file-name "denote-explore-network.html"
(file-name-directory (or load-file-name buffer-file-name)))))
(defcustom denote-explore-network-d3-js
"https://d3js.org/d3.v7.min.js"
"Location of the D3.js source code."
:group 'denote-explore
:type 'string)
(defcustom denote-explore-network-d3-colours
"schemeObservable10"
"Colour scheme for D3.js network visualisations.
Colours are assigned to fiule types in order of appearance in the JSON file.
Refer to URL `https://d3js.org/d3-scale-chromatic/categorical' for details."
:group 'denote-explore
:type '(choice
(const :tag "schemeCategory10" "schemeCategory10")
(const :tag "schemeAccent" "schemeAccent")
(const :tag "schemeDark2" "schemeDark2")
(const :tag "schemeObservable10" "schemeObservable10")
(const :tag "schemePaired" "schemePaired")
(const :tag "schemePastel1" "schemePastel1")
(const :tag "schemePastel2" "schemePastel2")
(const :tag "schemePastel3" "schemePastel3")
(const :tag "schemeSet1" "schemeSet1")
(const :tag "schemeSet2" "schemeSet2")
(const :tag "schemeTableau10" "schemeTableau10")))
(defcustom denote-explore-network-graphviz-header
'("layout=neato"
"size=20"
"ratio=compress"
"overlap=scale"
"sep=1"
"node[label=\"\" style=filled color=lightskyblue fillcolor=lightskyblue3
shape=circle fontsize=40 fontcolor=gray10 fontname = \"Helvetica, Arial, sans-serif\"]"
"edge[arrowsize=3 color=gray30]")
"List of strings for the header of a GraphViz DOT file.
Defines graph and layout properties and default edge and node attributes.
See graphviz.org for detailed documentation.
Properties for specific edges and nodes, as defined by the
`denote-explore-network-encode-graphviz' function, override these settings."
:group 'denote-explore
:package-version '(denote-explore . "1.3")
:type '(repeat string))
(defcustom denote-explore-network-graphviz-filetype "svg"
"Output file type for Denote GraphViz network files.
Use SVG or for interactivity (tootltips and hyperlinks).
See graphviz.org for detailed documentation."
:group 'denote-explore
:package-version '(denote-explore . "1.4")
:type '(choice
(const :tag "Scalable Vector Graphics (SVG)" "svg")
(const :tag "Portable Document Format (PDF)" "pdf")
(const :tag "Portable Network Graphics (PNG)" "png")
(string :tag "Other option")))
;;; INTERNAL VARIABLES
(defvar denote-explore-network-graph-formats
'((graphviz
:file-extension ".gv"
:encode denote-explore-network-encode-graphviz
:display denote-explore-network-display-graphviz)
(d3.js
:file-extension ".json"
:encode denote-explore-network-encode-json
:display denote-explore-network-display-json)
(gexf
:file-extension ".gexf"
:encode denote-explore-network-encode-gexf
:display nil))
"A-list of variables related to the network file formats.
Each element is of the form (SYMBOL . PROPERTY-LIST).
SYMBOL is one of those specified in `denote-explore-network-format'.
PROPERTY-LIST is a plist that consists of three elements:
- `:file-extension' File extension to save network.
- `:encode' function to encode network to graph type.
- `:display' function to display the graph in external software.")
(defvar denote-explore-graph-types
`(("Community"
:description "Notes matching a regular expression"
:generate denote-explore-network-community
:regenerate denote-explore-network-community-graph)
("Neighbourhood"
:description "Search n-deep in a selected note"
:generate denote-explore-network-neighbourhood
:regenerate denote-explore-network-neighbourhood-graph)
("Keywords"
:description "Relationships between keywords"
:generate denote-explore-network-keywords
:regenerate denote-explore-network-keywords-graph)
("Sequence"
:description "Hierarchical relationship between signatures"
:generate denote-explore-network-sequence
:regenerate denote-explore-network-sequence-graph))
"List of network types and their (re)generation functions.
PROPERTY-LIST is a plist that consists of three elements:
- `:description' Short explanation of graph type.
- `:generate': Function to select options and geenrate graph.
- `:regenerate': Function to regenerate graph from previous options.")
(define-obsolete-variable-alias
'denote-explore-load-directory
'denote-explore-network-filename "3.1")
(defvar denote-explore-network-previous
nil
"Store the previous network configuration to regenerate the last graph.
Parameters define the previous network, i.e.:
- `(\"keywords\")'
- `(\"neighbourhood\" \"20240101T084408\" 3)'
- `(\"community\" \"regex\")'
- `(\"sequence) \"root signature\"'")
;;; STATISTICS
;; Count number of notes, attachments and keywords
;;;###autoload
(defun denote-explore-count-notes (&optional attachments)
"Count number of Denote notes and attachments.
A note is defined by `denote-file-types', anything else is an attachment.
Count only ATTACHMENTS by prefixing with universal argument."
(interactive "P")
(let* ((all-files (length (denote-directory-files)))
(denote-files (length (denote-directory-files nil nil t)))
(attachment-files (- all-files denote-files)))
(if attachments
(message "%s attachments" attachment-files)
(message "%s notes (%s attachments)" denote-files attachment-files))))
;;;###autoload
(defun denote-explore-count-keywords ()
"Count distinct Denote keywords."
(interactive)
(let ((all-keywords (length (mapcan #'denote-extract-keywords-from-path
(denote-directory-files))))
(distinct-keywords (length (denote-keywords))))
(message "%s used keywords (%s distinct keywords)"
all-keywords distinct-keywords)))
;;;###autoload
(defun denote-explore-barchart-timeline ()
"Draw a column chart with the number of notes per year."
(interactive)
(let* ((files (denote-directory-files))
(file-names (mapcar #'file-name-nondirectory files))
;; Extract years from file names
(years (mapcar (lambda (file)
(substring file 0 4))
file-names))
(years-table (denote-explore--table years))
(years-table-sorted (sort years-table
(lambda (a b)
(string< (car a) (car b))))))
(denote-explore--barchart years-table-sorted
"Year"
"Denote notes and attachments timeline")))
;;; RANDOM WALKS
;; Jump to a random note, random linked note or random note with selected tag(s).
;; With universal argument the sample includes attachments.
(defun denote-explore--jump (files)
"Jump to a random note in the FILES file list.
Exclude riles matching regx in `denote-explore-random-regex-ignore'.
- `denote-explore-random-note': Jump to a random Denote file.
- `denote-explore-random-regex': Jump to a random Denote file that matches a
regular expression.
- `denote-explore-random-link': Jump to a random linked note (either forward or
backward) or attachments (forward only).
- `denote-explore-random-keyword': Jump to a random Denote file with the same
selected keyword(s)."
(let* ((ignore (if denote-explore-random-regex-ignore
(denote-directory-files denote-explore-random-regex-ignore)
nil))
(denote-sample (cl-set-difference files ignore :test 'string=)))
(find-file (nth (random (length denote-sample)) denote-sample))))
;;;###autoload
(defun denote-explore-random-note (&optional include-attachments)
"Jump to a random Denote file and optionally INCLUDE-ATTACHMENTS.
With universal argument the sample includes attachments."
(interactive "P")
(if-let ((denotes (denote-directory-files nil t (not include-attachments))))
(denote-explore--jump denotes)
(user-error "No Denote files found")))
;;;###autoload
(defun denote-explore-random-link (&optional attachments)
"Jump to a random linked from current buffer note.
With universal argument the sample includes links to ATTACHMENTS."
(interactive "P")
;; Gather links
(let* ((forward-links (denote-link-return-links))
(back-links (denote-link-return-backlinks))
(all-links (append forward-links back-links))
(links (if attachments
all-links
(seq-filter #'denote-file-is-note-p all-links))))
(if links
(denote-explore--jump links)
(message "Not a Denote file or no (back)links in or to this buffer"))))
(defun denote-explore--retrieve-keywords (file)
"Retrieve alphabetised list of keywords from Denote FILE.
Uses front matter for notes and the filename for attachments."
(let* ((file (if (eq file nil) "" file))
(filetype (denote-filetype-heuristics file))
(raw-keywords (if (denote-file-is-note-p file)
(denote-retrieve-keywords-value file filetype)
(denote-retrieve-filename-keywords file)))
(keywords (cond ((or (null raw-keywords) (equal raw-keywords "")) nil)
((stringp raw-keywords) (split-string raw-keywords "_"))
(t raw-keywords))))
(when keywords (sort keywords 'string<))))
(defun denote-explore--select-keywords ()
"Select Denote keyword(s) for random jump.
- Use \"*\" to select all listed keywords.
- If the current buffer has no Denote keywords, then choose from all available."
(let* ((raw-keywords (denote-explore--retrieve-keywords (buffer-file-name)))
(buffer-keywords (if (null raw-keywords)
(denote-keywords)
raw-keywords))
(keywords (if (> (length buffer-keywords) 1)
(delete-dups
(sort (completing-read-multiple
"Select keyword(s) (* selects all available keywords): "
buffer-keywords)
#'string<))
buffer-keywords)))
(if (string= (car keywords) "*") buffer-keywords keywords)))
;;;###autoload
(defun denote-explore-random-keyword (&optional include-attachments)
"Jump to a random note with selected keyword(s).
- Select one or more keywords from the active Denote buffer.
- Override the completion option by adding free text
- Use \"*\" to select all listed keywords.
Selecting multiple keywords requires `denote-sort-keywords' to be non-nil
or the target keywords are in the same order as the selection.
With universal argument the sample will INCLUDE-ATTACHMENTS."
(interactive "P")
(if-let* ((keyword-list (denote-explore--select-keywords))
(keyword-regex (concat "_" (mapconcat #'identity keyword-list ".*_"))))
(denote-explore-random-regex keyword-regex include-attachments)))
;;;###autoload
(defun denote-explore-random-regex (regex &optional include-attachments)
"Jump to a random not matching a regular expression REGEX.
Use Universal Argument to INCLUDE-ATTACHMENTS"
(interactive "sRegular expression: \nP")
(if-let ((sample (denote-directory-files regex t (not include-attachments))))
(denote-explore--jump sample)
(message "No matching Denote files found")))
;;; JANITOR
;; The Janitor provides various functions to maintain a Denote file collection.
(defun denote-explore--table (list)
"Generate an ordered frequency table from a LIST."
(sort (-frequencies list)
(lambda (a b) (> (cdr a) (cdr b)))))
(defun denote-explore--filter-duplicates (tally)
"Filter duplicates from an alist TALLY with frequencies.
Generate tally with `denote-explore-table'."
(mapcar #'car (cl-remove-if-not
(lambda (note)
(> (cdr note) 1))
tally)))
(defun denote-explore--duplicate-notes (exports)
"Find duplicate Denote files.
When EXPORTS, use complete filenames, else use Denote identifiers
and exclude exported Org files."
;; Count each unique identifier or filename
(let* ((files (denote-directory-files))
(filenames (mapcar (lambda (path)
(file-name-sans-extension
(file-name-nondirectory path)))
files))
(ids (mapcar #'denote-retrieve-filename-identifier files))
(id-count (denote-explore--table ids))
(duplicate-ids (denote-explore--filter-duplicates id-count))
(file-count (denote-explore--table filenames))
(duplicate-filenames (denote-explore--filter-duplicates file-count))
(duplicate-filenames-ids (mapcar #'denote-retrieve-filename-identifier
duplicate-filenames)))
(if exports
(cl-set-difference duplicate-ids duplicate-filenames-ids :test 'string=)
duplicate-ids)))
;;;###autoload
(defun denote-explore-duplicate-notes (&optional exclude-exports)
"Identify duplicate Denote IDs and EXCLUDE-EXPORTS.
If EXCLUDE-EXPORTS is nil, check Denote IDs, otherwise use file names without
extension. Using the universal argument excludes exported Denote files from
duplicate detection.
Duplicate files are displayed in a temporary buffer with links to the
suspected duplicates."
(interactive "P")
(message "Finding duplicated notes")
(if-let* ((duplicates (denote-explore--duplicate-notes exclude-exports)))
(with-current-buffer-window "*denote-duplicates*" nil nil
(erase-buffer)
(insert "#+title: Duplicate Denote files ")
(if exclude-exports
(insert "(Excluding exports)")
(insert "(Including exports)"))
(insert "\n#+date: ")
(org-insert-time-stamp (current-time) t t)
(insert "\n\n")
(if exclude-exports
(insert "Run without universal argument =C-u= to include exported files.\n")
(insert "Run with universal argument =C-u= to exclude exported files.\n"))
(dolist (id duplicates)
(insert (format "\n* Note ID [[denote:%s]]\n\n" id))
(dolist (filename (denote-directory-files id))
(insert (format " - [[file:%s][%s]]\n"
filename
(funcall denote-link-description-function filename)))))
(org-mode)
(read-only-mode))
(message "No duplicates found")))
(define-obsolete-function-alias
'denote-explore-identify-duplicate-identifiers
'denote-explore-identify-duplicate-notes "1.2")
(define-obsolete-function-alias
'denote-explore-identify-duplicate-notes
'denote-explore-duplicate-notes "3.3")
;;;###autoload
(defun denote-explore-duplicate-notes-dired (&optional filenames)
"Identify duplicate Denote IDs or FILENAMES.
If FILENAMES is nil, check Denote IDs, otherwise use complete file names.
Using the FILENAMES option (or using the universal argument) excludes
exported Denote files from duplicate-detection.
Duplicate files are displayed `find-dired'."
(interactive "P")
(if-let* ((duplicates (denote-explore--duplicate-notes filenames)))
(find-dired denote-directory
(mapconcat (lambda (id)
(format "-name '%s*'" id))
duplicates
" -o "))
(message "No duplicates found")))
(define-obsolete-function-alias
'denote-explore-identify-duplicate-notes-dired
'denote-explore-duplicate-notes-dired "3.3")
;;;###autoload
(defun denote-explore-single-keywords ()
"Select a note or attachment with a keyword that is only used once."
(interactive)
;; Count keywords and find singles
(if-let* ((keywords (mapcan #'denote-extract-keywords-from-path
(denote-directory-files)))
(keywords-count (denote-explore--table keywords))
(single-keywords (mapcar #'car (cl-remove-if-not
(lambda (note)
(= (cdr note) 1))
keywords-count)))
(selected-keyword (if single-keywords
(completing-read "Select single keyword: "
single-keywords))))
(find-file (car (denote-directory-files (concat "_" selected-keyword))))
(message "No single keywords in use.")))
;;;###autoload
(defun denote-explore-zero-keywords ()
"Select a note or attachment without any keywords."
(interactive)
;; Find all notes without an underscore
(if-let* ((zero-keywords (denote-directory-files "^[^_]*$")))
(find-file (completing-read "Select file with zero keywords: "
zero-keywords))
(message "All files have keywords.")))
(define-obsolete-function-alias
'denote-explore--alphabetical-p
nil "3.3")
(define-obsolete-function-alias
'denote-explore-sort-keywords
'denote-explore-sync-metadata "3.3")
;;;###autoload
(defun denote-explore-rename-keyword ()
"Rename or remove keyword(s) across the Denote collection.
Use an empty string as new keyword to remove the selection.
When selecting more than one existing keyword, all selections are renamed
to the new version or removed.
The filename is taken as the source of truth for attchments and the front matter
for notes.
All open Denote note buffers should be saved for this function to work reliably."
(interactive)
;; Save any open Denote files
(save-some-buffers nil #'(lambda ()
(denote-filename-is-note-p buffer-file-name)))
;; Select keywords and file candidates
(let* ((selected-keyword (denote-keywords-prompt "Keyword(s) to rename"))
(new-keyword (read-from-minibuffer "New keyword: "))
(keywords-regex (mapconcat
(lambda (keyword) (concat "_" keyword)) selected-keyword "\\|"))
(files (denote-directory-files keywords-regex)))
;; Loop through candidates
(dolist (file files)
(let* ((denote-rename-confirmations '(rewrite-front-matter modify-file-name))
(denote-sort-keywords t)
(denote-known-keywords nil)
(current-keywords (denote-explore--retrieve-keywords file))
(new-keywords (if (equal new-keyword "")
(cl-set-difference current-keywords selected :test 'string=)
(mapcar (lambda (keyword)
(if (member keyword selected-keyword) new-keyword keyword))
current-keywords)))
(file-type (denote-filetype-heuristics file)))
(denote-rename-file file
(denote-retrieve-title-or-filename file file-type)
(if (equal new-keywords nil) "" (delete-dups new-keywords))
(denote-retrieve-filename-signature file))))))
(define-obsolete-function-alias
'denote-explore--retrieve-title
'denote-retrieve-title-or-filename "1.4.2")
;;;###autoload
(defun denote-explore-sync-metadata ()
"Synchronise filenames with the metadata for all Denote notes.
The front matter is the source of truth. Keywords are saved alphabetically.
All open Denote note buffers need to be saved before invoking this function."
(interactive)
;; Save open Denote notes
(save-some-buffers nil #'(lambda ()
(denote-filename-is-note-p buffer-file-name)))
(let ((denote-rename-confirmations '(rewrite-front-matter modify-file-name))
(denote-sort-keywords t)
(notes (denote-directory-files nil nil t)))
;; Construct new file names and rename when different
(dolist (file notes)
(let* ((dir (file-name-directory file))
(file-type (denote-filetype-heuristics file))
(id (denote-retrieve-filename-identifier file))
(title (denote-retrieve-front-matter-title-value file file-type))
(signature (denote-retrieve-filename-signature file))
(keywords (denote-retrieve-front-matter-keywords-value file file-type))
(file-keywords (mapconcat #'identity keywords "_"))
(ext (file-name-extension file t))
(file-name (denote-format-file-name dir id keywords title ext signature)))
(if (not (string= file file-name))
(denote-rename-file-using-front-matter file))))
(message "Integrity check completed")))
;;;###autoload
(defun denote-explore-dead-links ()
"Check all Denote links in the `denote-directory'."
(interactive)
(message "Identifying dead links ...")
(if-let* ((files (denote-directory-files))
(ids (mapcar #'denote-retrieve-filename-identifier files))
(links (denote-explore--network-extract-edges files))
(dead-links (seq-filter
(lambda (link)
(not (member (cdr (assoc 'target link)) ids)))
links)))
(with-current-buffer-window "*Denote dead links*" nil nil
(org-mode)
(insert "#+title: List of dead Denote links\n")
(insert "#+date: ") (org-insert-time-stamp (current-time) t t)
(insert (format "\n\n%s dead links identified\n\n"
(length dead-links))
"Follow the hyperlinks to remove or repair dead links.\n\n"
"To disable confirmations, customise ~org-link-elisp-confirm-function~\n\n")
(insert "| Source | Target |\n")
(insert "|--------|--------|\n")
(dolist (link dead-links)
(let* ((source (cdr (assoc 'source link)))
(target (cdr (assoc 'target link)))
(file (car (denote-directory-files source)))
(file-type (denote-filetype-heuristics file))
(title (denote-retrieve-front-matter-title-value file file-type)))
(insert "|")
(insert "[[elisp:(denote-explore--review-dead-link \""
source "\" \"" target "\")][" title "]]")
(insert "|" target "|\n")))
(org-table-align))
(message "No dead Denote links found")))
(defun denote-explore--review-dead-link (source target)
"Jump to the location of a dead link to TARGET found in SOURCE."
(let ((file (car (denote-directory-files source))))
(find-file file)
(org-toggle-link-display)
(search-forward-regexp target)))
;;; VISUALISATION
;; Bar charts
;; Leverages the built-in chart package for plain text visualisation.
(defun denote-explore--barchart (table var title &optional n horizontal)
"Create a barchart from a frequency TABLE with top N entries.
VAR and TITLE used for display."
(chart-bar-quickie
(if horizontal 'horizontal 'vertical)
title
(mapcar #'car table) var
(mapcar #'cdr table) "Frequency" n))
;;;###autoload
(defun denote-explore-barchart-keywords (n)
"Create a barchart with the top N most used Denote keywords."
(interactive "nNumber of keywords: ")
(let* ((keywords (mapcan #'denote-extract-keywords-from-path
(denote-directory-files)))
(keywords-table (denote-explore--table keywords)))
(denote-explore--barchart
keywords-table
(concat "Top-" (number-to-string n) " Denote Keywords")
(denote-explore-count-keywords) n)))
(define-obsolete-function-alias
'denote-explore-keywords-barchart
'denote-explore-barchart-keywords "3.0")
;;;###autoload
(defun denote-explore-barchart-filetypes (&optional attachments)
"Visualise the Denote file types for notes and/or attachments.
With universal argument only visualises ATTACHMENTS, which excludes file
types listed in `denote-file-type-extensions'."
(interactive "P")
(let* ((files (if attachments
(cl-remove-if #'denote-file-is-note-p (denote-directory-files))
(denote-directory-files)))
(extensions (mapcar (lambda(file) (file-name-extension file))
files))
(ext-list (if attachments
(cl-set-difference extensions
(denote-file-type-extensions)
:test 'equal)
extensions))
(title (if attachments
(denote-explore-count-notes :attachments)
(denote-explore-count-notes))))
(denote-explore--barchart
(denote-explore--table ext-list) "Denote file extensions" title)))
(define-obsolete-function-alias
'denote-explore-extensions-barchart
'denote-explore-barchart-filetypes "3.0")
(defun denote-explore--network-sum-degrees (nodes)
"Sum the degrees in NODES, producing a new alist with degree counts."
(let ((degree-sums ()))
(dolist (entry nodes degree-sums)
(let* ((degree (cdr (assoc 'degree entry)))
(current-count (cdr (assoc degree degree-sums))))
(if current-count
(setcdr (assoc degree degree-sums) (1+ current-count))
(push (cons degree 1) degree-sums))))
(sort degree-sums (lambda (a b) (< (car a) (car b))))))
;;;###autoload
(defun denote-explore-barchart-degree (&optional text-only)
"Visualise the degree for each Denote file (total links and backlinks).
The universal argument includes TEXT-ONLY files in the analyis."
(interactive "P")
(message "Analysing Denote network ...")
(let* ((graph (denote-explore-network-community-graph "" text-only))
(nodes (cdr (assoc 'nodes graph)))
(degrees (denote-explore--network-sum-degrees nodes))
(txt-degrees (mapcar (lambda (pair)
(cons (number-to-string (car pair)) (cdr pair)))
degrees)))
(denote-explore--barchart txt-degrees "Degree" "Node degree distribution")))
(define-obsolete-function-alias
'denote-explore-degree-barchart
'denote-explore-barchart-degree "3.0")
;;;###autoload
(defun denote-explore-barchart-backlinks (n)
"Visualise the number of backlinks for N nodes in the Denote network."
(interactive "nNumber of nodes: ")
(message "Analysing Denote network ...")
(let* ((graph (denote-explore-network-community-graph "" t))
(nodes (cdr (assoc 'nodes graph)))
(backlinks (mapcar (lambda (node)
(cons
(cdr (assq 'name node))
(cdr (assq 'backlinks node))))
nodes)))
(sort backlinks (lambda (a b)
(> (cdr a) (cdr b))))
(denote-explore--barchart backlinks "Backlinks" "Node backlinks distribution" n t)))
(define-obsolete-function-alias
'denote-explore-backlinks-barchart
'denote-explore-barchart-backlinks "3.0")
(defun denote-explore--idenitfy-isolated (&optional text-only)
"Identify Denote files without (back)links.
Using the universal argument provides TEXT-ONLY files (excludes attachments)."
(let* ((files (denote-directory-files nil nil text-only))
(all-ids (mapcar #'denote-retrieve-filename-identifier files))
(edges (denote-explore--network-extract-edges files))
(linked-ids (denote-explore--network-extract-unique-nodes edges))
(isolated-ids (seq-remove (lambda (id) (member id linked-ids)) all-ids)))
(-map (lambda (id) (-filter (lambda (f) (string-match-p id f)) files))
isolated-ids)))
;;;###autoload
(defun denote-explore-isolated-files (&optional text-only)
"Identify Denote files without (back)links.
Using the universal argument excludes attachments (TEXT-ONLY)."
(interactive "P")
(message "Searching for isolated files ...")
(let ((isolated (denote-explore--idenitfy-isolated text-only)))
(find-file (completing-read "Select isolated file: " isolated))))
;;; DEFINE GRAPHS
;; Define various graph types as an association list
;; Community graph
(define-obsolete-function-alias
'denote-explore--network-zip-alists
'nil "3.3")
(defun denote-explore--network-extract-edges (files)
"Extract forward links as network edges from FILES."
(let* ((text-files (seq-filter #'denote-file-is-note-p files))
(links-xref (xref-matches-in-files
"\\[denote:[0-9]\\{8\\}T[0-9]\\{6\\}" text-files))
(source (mapcar #'denote-retrieve-filename-identifier
(mapcar
#'xref-location-group
(mapcar #'xref-match-item-location
links-xref))))
(links (mapcar #'substring-no-properties
(mapcar #'xref-match-item-summary
links-xref)))
(target (mapcar
(lambda (str)
(when (string-match denote-id-regexp str)
(match-string 0 str)))
links))
;; Zip this lists as an alist (((source . "a") (target "b")) (...))
(all-edges (-map (lambda (pair)
`((source . ,(car pair)) (target . ,(cdr pair))))
(-zip-pair source target))))
all-edges))
(define-obsolete-function-alias
'denote-explore--extract-vertices
'denote-explore--network-extract-node "1.2")
(defun denote-explore--network-extract-node (file)
"Extract metadata for note or attachment FILE."
(when (file-exists-p file)
(let ((id (denote-retrieve-filename-identifier file))
(signature (denote-retrieve-filename-signature file))
(name (denote-retrieve-title-or-filename
file (denote-filetype-heuristics file)))
(keywords (denote-retrieve-filename-keywords file))
(type (file-name-extension file)))
(setq keywords (if keywords (string-split keywords "_") ""))
`((id . ,id) (signature . ,signature) (name . ,name)
(keywords . ,keywords) (type . ,type) (filename . ,file)))))
(defun denote-explore--network-prune-edges (nodes edges)
"Select EDGES (links) where both source and target are part of NODES.
Prunes any edges that link to outside a community of NODES."
(let ((filtered-edges '()))
(dolist (edge edges filtered-edges)
(let ((source (cdr (assoc 'source edge)))
(target (cdr (assoc 'target edge))))
(when (and (member source nodes) (member target nodes))
(push edge filtered-edges))))
(nreverse filtered-edges)))
(defun denote-explore--network-count-edges (edges)
"Count occurrences of EDGES to set their weight.
The result is an association list of all edges and their weights:
`((source . id1) (source . id2) (weight . w))'."
(let* ((edges-count (-frequencies edges))
(edges-alist (mapcar (lambda (item)
(let ((source (cdr (assoc 'source (car item))))
(target (cdr (assoc 'target (car item))))
(weight (cdr item)))
`((source . ,source)
(target . ,target)
(weight . ,weight))))
edges-count)))
edges-alist))
(defun denote-explore--network-degree (nodes edges)
"Calculate the degree of nodes in a network graph NODES and EDGES.
The degree of a Denote graph node is defined by the sum of links and backlinks
of a file."
(mapcar (lambda (node)
(let ((node-id (cdr (assoc 'id node)))
(degree 0))
(dolist (edge edges)
(when (or (string= node-id (cdr (assoc 'source edge)))
(string= node-id (cdr (assoc 'target edge))))
(setq degree (+ degree 1))))
(append node (list (cons 'degree degree)))))
nodes))
(defun denote-explore--network-backlinks (nodes edges)
"Calculate the number of backlinks for NODES and EDGES."
(mapcar (lambda (node)
(let ((node-id (cdr (assoc 'id node)))
(backlinks 0))
(dolist (edge edges)
(when (string= node-id (cdr (assoc 'target edge)))
(setq backlinks (+ backlinks (cdr (assoc 'weight edge))))))
(append node (list (cons 'backlinks backlinks)))))
nodes))
(defun denote-explore--network-filter-files (files)
"Remove files matching `denote-explore-network-regex-ignore' from Denote FILES.
Removes selected files from neighbourhood or community visualisation."
(let ((ignore (if denote-explore-network-regex-ignore
(denote-directory-files denote-explore-network-regex-ignore)
nil)))
(cl-set-difference files ignore :test 'string=)))
(defun denote-explore-network-community-graph (regex &optional text-only)
"Generate network community association list for note matching REGEX.
Optionally include TEXT-ONLY files (no attachments).
Links to notes outside the search area are pruned."
(if-let* ((files (denote-explore--network-filter-files
(denote-directory-files regex nil text-only)))
(ids (mapcar #'denote-extract-id-from-string files))
(edges (denote-explore--network-extract-edges files))
(edges-pruned (denote-explore--network-prune-edges ids edges))
(edges-alist (denote-explore--network-count-edges edges-pruned))
(nodes (mapcar #'denote-explore--network-extract-node files))
(nodes-degrees (denote-explore--network-degree nodes edges-alist))
(nodes-alist (denote-explore--network-backlinks nodes-degrees edges-alist))
(meta-alist `((directed . t) (type . "Community") (parameters ,regex))))
`((meta . ,meta-alist) (nodes . ,nodes-alist) (edges . ,edges-alist))
(user-error "No Denote files or (back)links found for %s" regex)))
(defun denote-explore-network-community (&optional text-only)
"Define inputs for a network community and generate graph.
Optionally include TEXT-ONLY files."
(let ((regex (read-from-minibuffer
"Enter regular expression (empty string for all notes):")))
(setq denote-explore-network-previous `("Community" ,regex))
(message "Building graph for \"%s\" community " regex)
(denote-explore-network-community-graph regex text-only)))
;;; keywords graph
(defun denote-explore--network-keywords-extract (files)
"Convert keywords from FILES to a list of lists.
Notes with only one keyword and keywords listed in
`denote-explore-network-keywords-ignore' are ignored."
(let ((keywords (mapcar #'denote-retrieve-filename-keywords files))
(processed-keywords '()))
(dolist (keyword keywords processed-keywords)
(when keyword
(let* ((split-keywords (split-string keyword "_"))
(filtered-keywords
(seq-remove
(lambda (k)
(member k denote-explore-network-keywords-ignore))
split-keywords)))
(when (> (length filtered-keywords) 1)
(push filtered-keywords processed-keywords)))))
(nreverse processed-keywords)))
(defun denote-explore--network-keyword-edges (keywords)
"Generate complete graph from list of KEYWORDS.
In a complete graph (network), all nodes are connected to each other."
(let ((network '())
(length (length keywords)))
(dotimes (i length)
(dotimes (j length)
(unless (or (= i j) (> i j)) ; Avoid duplicate and reversed pairs
(let ((source (nth i keywords))
(target (nth j keywords)))
(push (list (cons 'source source)
(cons 'target target)) network)))))
network))
(defun denote-explore-network-keywords (&optional text-only)
"Enter a positive integer for a minimum weight and generate a keywords graph.
Optionally analyse TEXT-ONLY files."
(let ((min-weight (read-number "Enter min-weight (integer > 0): " 1)))
(while (<= min-weight 0)
(setq min-weight (read-number "Invalid input. Enter an integer > 0: " 1)))
(setq denote-explore-network-previous `("Keywords" ,min-weight))
(denote-explore-network-keywords-graph min-weight text-only)))
(defun denote-explore--network-keywords-flatten-edges (edges)
"Flatten list of network EDGES."
(let ((edges-alist '()))
(dolist (sublist edges)
(dolist (edge sublist)
(push edge edges-alist)))
edges-alist))
(defun denote-explore-network-keywords-graph (min-weight text-only)
"Generate Denote graph object from keywords with MIN-WEIGHT edges.
Optionally analyse TEXT-ONLY files."
(let* ((files (denote-directory-files nil nil text-only))
(keywords (denote-explore--network-keywords-extract files))
(edges (mapcar #'denote-explore--network-keyword-edges keywords))
(all-edges-alist (denote-explore--network-count-edges
(denote-explore--network-keywords-flatten-edges edges)))
(edges-alist (cl-remove-if-not
(lambda (edge) (>= (cdr (assoc 'weight edge)) min-weight))
all-edges-alist))
(unique-nodes (denote-explore--network-extract-unique-nodes edges-alist))
(nodes '())
(nodes (dolist (node unique-nodes nodes) ; Iterating over nodes
(push (list (cons 'id node) (cons 'name node)) nodes)))
(nodes-alist (denote-explore--network-degree (nreverse nodes) edges-alist))
(meta-alist `((directed . nil) (type . "Keywords") (parameters . ,min-weight))))
`((meta . ,meta-alist) (nodes . ,nodes-alist) (edges . ,edges-alist))))
;;; Neighbourhood Graph
(defun denote-explore--network-extract-unique-nodes (edges-alist)
"Extract all unique `source' and `target' nodes from EDGES-ALIST."
(delete-dups
(mapcan (lambda (entry)
(list (cdr (assoc 'source entry))
(cdr (assoc 'target entry))))
edges-alist)))
(define-obsolete-function-alias
'denote-explore--network-find-edges nil "3.2.1")
(defun denote-explore--network-unique-edges (edges-a edges-b)
"Return a list of unique edges from EDGES-A and EDGES-B."
(let ((edges (append edges-a edges-b))
(seen '())
(unique '()))
(dolist (edge edges)
(unless (member edge seen)
(push edge seen)
(push edge unique)))
unique))
(defun denote-explore--network-neighbourhood-edges (file depth text-only)
"Itteratively Find all links to and from FILE up to DEPTH or less.
Optionally include TEXT-ONLY files."