Skip to content

Commit de3ac32

Browse files
arkanarkan
committed
Cache vault index for Dataview
1 parent ea20520 commit de3ac32

4 files changed

Lines changed: 213 additions & 28 deletions

File tree

‎internal/app/app.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,20 @@ import (
1717
"regexp"
1818
"sort"
1919
"strings"
20+
"sync"
2021
"time"
2122

2223
"gopkg.in/yaml.v3"
2324
)
2425

2526
type (
26-
Vault struct{ Root string }
27-
Note struct {
27+
Vault struct {
28+
Root string
29+
indexMu sync.Mutex
30+
indexCache *VaultIndex
31+
indexCacheKey string
32+
}
33+
Note struct {
2834
Path, RelPath, Text, Body string
2935
Frontmatter map[string]any
3036
ModTime time.Time

‎internal/app/dataview.go‎

Lines changed: 39 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -79,12 +79,25 @@ func RenderDataviewBlockWithIndex(v *Vault, idx *VaultIndex, raw string) templat
7979

8080
func preprocessDataviewBlocks(s string, v *Vault) string {
8181
re := regexp.MustCompile("(?s)```dataview\\s*\\n(.*?)\\n```")
82+
if !re.MatchString(s) {
83+
return s
84+
}
85+
idx, err := v.BuildIndex()
86+
if err != nil {
87+
return re.ReplaceAllStringFunc(s, func(m string) string {
88+
parts := re.FindStringSubmatch(m)
89+
if len(parts) != 2 {
90+
return m
91+
}
92+
return string(dataviewError(parts[1], err))
93+
})
94+
}
8295
return re.ReplaceAllStringFunc(s, func(m string) string {
8396
parts := re.FindStringSubmatch(m)
8497
if len(parts) != 2 {
8598
return m
8699
}
87-
return string(RenderDataviewBlock(v, parts[1]))
100+
return string(RenderDataviewBlockWithIndex(v, idx, parts[1]))
88101
})
89102
}
90103

@@ -288,11 +301,12 @@ func splitTopLevel(s string, sep rune) []string {
288301

289302
func evalDataviewRows(v *Vault, idx *VaultIndex, q dataviewQuery) ([]dataviewRow, error) {
290303
var rows []dataviewRow
304+
heavy := q.requiredHeavyFields()
291305
if q.Kind == "TASK" {
292306
for _, meta := range idx.Notes {
293307
if sourceMatches(meta, q.From) {
294308
for _, task := range extractTasksForNote(v, meta) {
295-
r := dataviewRow{Note: &meta, Task: &task, Data: dataviewBaseData(v, idx, meta)}
309+
r := dataviewRow{Note: &meta, Task: &task, Data: dataviewBaseData(v, idx, meta, heavy)}
296310
if whereMatches(r, q.Where) {
297311
rows = append(rows, r)
298312
}
@@ -302,7 +316,7 @@ func evalDataviewRows(v *Vault, idx *VaultIndex, q dataviewQuery) ([]dataviewRow
302316
} else {
303317
for _, meta := range idx.Notes {
304318
if sourceMatches(meta, q.From) {
305-
r := dataviewRow{Note: &meta, Data: dataviewBaseData(v, idx, meta)}
319+
r := dataviewRow{Note: &meta, Data: dataviewBaseData(v, idx, meta, heavy)}
306320
if whereMatches(r, q.Where) {
307321
rows = append(rows, r)
308322
}
@@ -322,26 +336,31 @@ func evalDataviewRows(v *Vault, idx *VaultIndex, q dataviewQuery) ([]dataviewRow
322336
return rows, nil
323337
}
324338

325-
func dataviewBaseData(v *Vault, idx *VaultIndex, meta NoteMeta) map[string]any {
339+
type dataviewHeavyFields struct{ Content, Inlinks bool }
340+
341+
func (q dataviewQuery) requiredHeavyFields() dataviewHeavyFields {
342+
var exprs []string
343+
for _, c := range q.Columns {
344+
exprs = append(exprs, c.Expr)
345+
}
346+
for _, s := range q.Sorts {
347+
exprs = append(exprs, s.Expr)
348+
}
349+
exprs = append(exprs, q.Where, q.GroupBy, q.Flatten)
350+
joined := strings.Join(exprs, "\n")
351+
return dataviewHeavyFields{Content: strings.Contains(joined, "file.content"), Inlinks: strings.Contains(joined, "file.inlinks")}
352+
}
353+
354+
func dataviewBaseData(v *Vault, idx *VaultIndex, meta NoteMeta, heavy dataviewHeavyFields) map[string]any {
326355
data := map[string]any{}
327-
if n, err := v.ReadNote(meta.RelPath); err == nil {
328-
data["file.content"] = n.Body
329-
}
330-
var inlinks []dataviewLink
331-
stem := strings.TrimSuffix(filepath.Base(meta.RelPath), filepath.Ext(meta.RelPath))
332-
noExt := strings.TrimSuffix(meta.RelPath, filepath.Ext(meta.RelPath))
333-
for _, other := range idx.Notes {
334-
if other.RelPath == meta.RelPath {
335-
continue
336-
}
337-
for _, target := range other.OutgoingWikiLinks {
338-
if target == stem || target == noExt || target == meta.RelPath || strings.TrimSuffix(target, filepath.Ext(target)) == noExt {
339-
inlinks = append(inlinks, dataviewLink{URL: other.URL, Text: noteFileName(other)})
340-
break
341-
}
356+
if heavy.Content {
357+
if n, err := v.ReadNote(meta.RelPath); err == nil {
358+
data["file.content"] = n.Body
342359
}
343360
}
344-
data["file.inlinks"] = inlinks
361+
if heavy.Inlinks {
362+
data["file.inlinks"] = idx.Inlinks[meta.RelPath]
363+
}
345364
return data
346365
}
347366

‎internal/app/dataview_test.go‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,90 @@ FROM "Projects"`))
230230
}
231231
}
232232

233+
func TestVaultIndexCacheReusesUnchangedIndexAndInvalidatesOnMarkdownChange(t *testing.T) {
234+
v := makeDataviewVault(t)
235+
idx1, err := v.BuildIndex()
236+
if err != nil {
237+
t.Fatal(err)
238+
}
239+
idx2, err := v.BuildIndex()
240+
if err != nil {
241+
t.Fatal(err)
242+
}
243+
if idx1 != idx2 {
244+
t.Fatalf("unchanged vault should reuse cached index pointer")
245+
}
246+
247+
writeDataviewFixture(t, v, "Projects/New.md", "---\ntitle: New\nstatus: active\n---\n# New\n")
248+
idx3, err := v.BuildIndex()
249+
if err != nil {
250+
t.Fatal(err)
251+
}
252+
if idx3 == idx2 {
253+
t.Fatalf("changed vault should invalidate cached index")
254+
}
255+
if _, ok := idx3.ByRel["Projects/New.md"]; !ok {
256+
t.Fatalf("invalidated index should include new note")
257+
}
258+
}
259+
260+
func TestDataviewRowsComputeHeavyFieldsOnlyWhenQueryNeedsThem(t *testing.T) {
261+
v := makeDataviewVault(t)
262+
writeDataviewFixture(t, v, "Projects/Linked.md", "# Linked\n[[Alpha]]\n")
263+
idx, err := v.BuildIndex()
264+
if err != nil {
265+
t.Fatal(err)
266+
}
267+
268+
lightQuery, err := parseDataviewQuery(`TABLE file.link, status FROM "Projects" WHERE status = "active" SORT file.name`)
269+
if err != nil {
270+
t.Fatal(err)
271+
}
272+
lightRows, err := evalDataviewRows(v, idx, lightQuery)
273+
if err != nil {
274+
t.Fatal(err)
275+
}
276+
if len(lightRows) == 0 {
277+
t.Fatal("expected light dataview rows")
278+
}
279+
for _, row := range lightRows {
280+
if _, ok := row.Data["file.content"]; ok {
281+
t.Fatalf("light query should not populate file.content: %#v", row.Data)
282+
}
283+
if _, ok := row.Data["file.inlinks"]; ok {
284+
t.Fatalf("light query should not populate file.inlinks: %#v", row.Data)
285+
}
286+
}
287+
288+
heavyQuery, err := parseDataviewQuery(`TABLE file.inlinks FROM "Projects" WHERE contains(file.content, "Alpha")`)
289+
if err != nil {
290+
t.Fatal(err)
291+
}
292+
heavyRows, err := evalDataviewRows(v, idx, heavyQuery)
293+
if err != nil {
294+
t.Fatal(err)
295+
}
296+
if len(heavyRows) == 0 {
297+
t.Fatal("expected heavy dataview rows")
298+
}
299+
foundAlpha := false
300+
for _, row := range heavyRows {
301+
if row.Note != nil && row.Note.RelPath == "Projects/Alpha.md" {
302+
foundAlpha = true
303+
if _, ok := row.Data["file.content"]; !ok {
304+
t.Fatalf("query using file.content should populate it")
305+
}
306+
inlinks, ok := row.Data["file.inlinks"].([]dataviewLink)
307+
if !ok || len(inlinks) == 0 || inlinks[0].Text != "Linked" {
308+
t.Fatalf("query using file.inlinks should use precomputed backlinks, got %#v", row.Data["file.inlinks"])
309+
}
310+
}
311+
}
312+
if !foundAlpha {
313+
t.Fatalf("expected Alpha row in heavy query: %#v", heavyRows)
314+
}
315+
}
316+
233317
func TestDataviewTableClientEnhancementAssetsArePresent(t *testing.T) {
234318
for _, want := range []string{".dataview-table-wrap", ".dataview-error", "overflow-x:auto", ".dataview-filter", ".dataview-pager"} {
235319
if !strings.Contains(css, want) {

‎internal/app/index.go‎

Lines changed: 82 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ package app
22

33
import (
44
"fmt"
5+
"os"
6+
"path/filepath"
57
"regexp"
68
"sort"
79
"strconv"
@@ -28,17 +30,31 @@ type WikiLinkOccurrence struct {
2830
}
2931

3032
type VaultIndex struct {
31-
Notes []NoteMeta
32-
ByRel map[string]NoteMeta
33-
Tags map[string][]NoteMeta
33+
Notes []NoteMeta
34+
ByRel map[string]NoteMeta
35+
Tags map[string][]NoteMeta
36+
Inlinks map[string][]dataviewLink
3437
}
3538

3639
func (v *Vault) BuildIndex() (*VaultIndex, error) {
40+
v.indexMu.Lock()
41+
defer v.indexMu.Unlock()
42+
43+
files := v.MarkdownFiles()
44+
cacheKey, err := vaultIndexCacheKey(v, files)
45+
if err != nil {
46+
return nil, err
47+
}
48+
if v.indexCache != nil && v.indexCacheKey == cacheKey {
49+
return v.indexCache, nil
50+
}
51+
3752
idx := &VaultIndex{
38-
ByRel: map[string]NoteMeta{},
39-
Tags: map[string][]NoteMeta{},
53+
ByRel: map[string]NoteMeta{},
54+
Tags: map[string][]NoteMeta{},
55+
Inlinks: map[string][]dataviewLink{},
4056
}
41-
for _, p := range v.MarkdownFiles() {
57+
for _, p := range files {
4258
note, err := v.ReadNote(p)
4359
if err != nil {
4460
return nil, err
@@ -63,9 +79,69 @@ func (v *Vault) BuildIndex() (*VaultIndex, error) {
6379
for tag := range idx.Tags {
6480
sort.Slice(idx.Tags[tag], func(i, j int) bool { return idx.Tags[tag][i].RelPath < idx.Tags[tag][j].RelPath })
6581
}
82+
idx.Inlinks = buildDataviewInlinks(idx)
83+
v.indexCache = idx
84+
v.indexCacheKey = cacheKey
6685
return idx, nil
6786
}
6887

88+
func vaultIndexCacheKey(v *Vault, files []string) (string, error) {
89+
var b strings.Builder
90+
for _, p := range files {
91+
st, err := os.Stat(p)
92+
if err != nil {
93+
return "", err
94+
}
95+
b.WriteString(v.Rel(p))
96+
b.WriteByte('\x00')
97+
b.WriteString(strconv.FormatInt(st.ModTime().UnixNano(), 10))
98+
b.WriteByte(':')
99+
b.WriteString(strconv.FormatInt(st.Size(), 10))
100+
b.WriteByte('\n')
101+
}
102+
return b.String(), nil
103+
}
104+
105+
func buildDataviewInlinks(idx *VaultIndex) map[string][]dataviewLink {
106+
keys := map[string][]string{}
107+
for _, meta := range idx.Notes {
108+
stem := strings.TrimSuffix(filepath.Base(meta.RelPath), filepath.Ext(meta.RelPath))
109+
noExt := strings.TrimSuffix(meta.RelPath, filepath.Ext(meta.RelPath))
110+
for _, key := range []string{meta.RelPath, noExt, stem} {
111+
if key == "" {
112+
continue
113+
}
114+
keys[key] = append(keys[key], meta.RelPath)
115+
}
116+
}
117+
118+
inlinks := map[string][]dataviewLink{}
119+
seen := map[string]bool{}
120+
for _, source := range idx.Notes {
121+
for _, target := range source.OutgoingWikiLinks {
122+
for _, key := range []string{target, strings.TrimSuffix(target, filepath.Ext(target))} {
123+
for _, destRel := range keys[key] {
124+
if destRel == source.RelPath {
125+
continue
126+
}
127+
seenKey := source.RelPath + "\x00" + destRel
128+
if seen[seenKey] {
129+
continue
130+
}
131+
seen[seenKey] = true
132+
inlinks[destRel] = append(inlinks[destRel], dataviewLink{URL: source.URL, Text: noteFileName(source)})
133+
}
134+
}
135+
}
136+
}
137+
for rel := range inlinks {
138+
sort.Slice(inlinks[rel], func(i, j int) bool {
139+
return inlinks[rel][i].Text < inlinks[rel][j].Text
140+
})
141+
}
142+
return inlinks
143+
}
144+
69145
func extractTags(note Note) []string {
70146
seen := map[string]bool{}
71147
var tags []string

0 commit comments

Comments
 (0)