Skip to content

Commit 98424b5

Browse files
committed
feat(docs-pipe): preserve underscores in tokens + heading-match boost for retrieval
Underscores no longer split identifiers, so try_ai/db_open/... stay single distinctive terms. Query terms matching a chunk's heading get a 3x weight, surfacing the defining section (e.g. 'try_ai — AI Self-Healing') above chunks that merely mention the term.
1 parent 92f67dc commit 98424b5

1 file changed

Lines changed: 10 additions & 5 deletions

File tree

docs-pipe/module.pipe

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,7 @@ fn load_cache idx
175175
set item "line_end" (get r "line_end")
176176
set item "text" (get r "content")
177177
set item "text_lower" (lower (get r "content"))
178+
set item "heading_lower" (lower (get r "heading"))
178179
set item "lang" (get r "lang")
179180
set item "vec" (parse_json (get r "vec"))
180181
push cache item
@@ -267,7 +268,7 @@ fn is_stop t
267268

268269
fn tokenize text
269270
low: lower text
270-
cleaned: regex_replace "[^a-z0-9äöüß]+" " " low
271+
cleaned: regex_replace "[^a-z0-9äöüß_]+" " " low
271272
parts: split cleaned " "
272273
out: []
273274
i: 0
@@ -292,13 +293,17 @@ fn keyword_denom terms df n
292293
i: i + 1
293294
sum
294295

295-
fn keyword_score text_lower terms df n
296+
fn keyword_score text_lower heading_lower terms df n
296297
score: 0
297298
i: 0
298299
while i < (len terms)
299300
t: at terms i
300-
if contains text_lower t
301-
score: score + (term_weight t df n)
301+
w: term_weight t df n
302+
if contains heading_lower t
303+
score: score + (w * 3)
304+
else
305+
if contains text_lower t
306+
score: score + w
302307
i: i + 1
303308
score
304309

@@ -351,7 +356,7 @@ export fn doc_search idx query k
351356
sem: cosine_sim qvec (get c "vec")
352357
kw: 0
353358
if kw_denom > 0
354-
kw: (keyword_score (get c "text_lower") terms df n) / kw_denom
359+
kw: (keyword_score (get c "text_lower") (get c "heading_lower") terms df n) / kw_denom
355360
score: (sem * sem_w) + (kw * kw_w)
356361
push scored {c: c, score: score}
357362
i: i + 1

0 commit comments

Comments
 (0)