Skip to content

lmdb: fulltext query engine #1202

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
v0l wants to merge 2 commits into
base: master
Choose a base branch
from
Open

lmdb: fulltext query engine #1202

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3385eef
lmdb: fulltext query engine
v0l Jan 2, 2026
ca93ba9
lmdb: cache parsed search ast
v0l Jan 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions database/nostr-lmdb/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ nostr = { workspace = true, features = ["std"] }
nostr-database = { workspace = true, features = ["flatbuf"] }
tokio = { workspace = true, features = ["sync"] }
tracing.workspace = true
tantivy-query-grammar = "0.25"

[target.'cfg(not(all(target_os = "macos", target_os = "ios")))'.dependencies]
heed = { version = "0.20", default-features = false, features = ["read-txn-no-tls"] }
Expand Down
149 changes: 8 additions & 141 deletions database/nostr-lmdb/src/store/filter.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,16 @@ use std::collections::{BTreeMap, BTreeSet, HashSet};

use nostr::event::borrow::EventBorrow;
use nostr::{Filter, SingleLetterTag, Timestamp};
use tantivy_query_grammar::{parse_query, UserInputAst};

const TITLE: &str = "title";
const DESCRIPTION: &str = "description";
const SUBJECT: &str = "subject";
const NAME: &str = "name";
use super::query::eval_ast;

pub(crate) struct DatabaseFilter {
pub(crate) ids: HashSet<[u8; 32]>,
pub(crate) authors: HashSet<[u8; 32]>,
pub(crate) kinds: HashSet<u16>,
// THIS IS LOWERCASE
pub(crate) search: Option<String>,
/// Pre-parsed search query AST (query string already lowercased before parsing)
pub(crate) search: Option<UserInputAst>,
pub(crate) since: Option<Timestamp>,
pub(crate) until: Option<Timestamp>,
pub(crate) generic_tags: BTreeMap<SingleLetterTag, BTreeSet<String>>,
Expand Down Expand Up @@ -68,28 +66,7 @@ impl DatabaseFilter {
#[inline]
fn search_match(&self, event: &EventBorrow) -> bool {
match &self.search {
Some(query) => {
// NOTE: `query` was already converted to lowercase
let query: &[u8] = query.as_bytes();

// Match content - early return on match
if match_content(query, event.content.as_bytes()) {
return true;
}

// Match tags - only if content didn't match
for (kind, content) in event
.tags
.iter()
.filter_map(|t| Some((t.kind(), t.content()?)))
{
if is_allowed_tag_kind(kind) && match_content(query, content.as_bytes()) {
return true;
}
}

false
}
Some(ast) => eval_ast(ast, event),
None => true,
}
}
Expand All @@ -106,36 +83,6 @@ impl DatabaseFilter {
}
}

#[inline]
fn match_content(query: &[u8], content: &[u8]) -> bool {
// Early exit if query is empty
if query.is_empty() {
return false;
}

// Early exit for impossible matches
if query.len() > content.len() {
return false;
}

// Fast path for single-byte searches (common case)
if query.len() == 1 {
let query_byte = query[0];
return content
.iter()
.any(|&b| b.to_ascii_lowercase() == query_byte);
}

content
.windows(query.len())
.any(|window| window.eq_ignore_ascii_case(query))
}

#[inline]
fn is_allowed_tag_kind(kind: &str) -> bool {
matches!(kind, TITLE | DESCRIPTION | SUBJECT | NAME)
}

impl From<Filter> for DatabaseFilter {
fn from(filter: Filter) -> Self {
Self {
Expand All @@ -156,94 +103,14 @@ impl From<Filter> for DatabaseFilter {
.kinds
.map(|kinds| kinds.into_iter().map(|id| id.as_u16()).collect())
.unwrap_or_default(),
search: filter.search.map(|mut s| {
// Convert to lowercase
search: filter.search.and_then(|mut s| {
// Convert to lowercase and parse into AST
s.make_ascii_lowercase();
s
parse_query(&s).ok()
}),
since: filter.since,
until: filter.until,
generic_tags: filter.generic_tags,
}
}
}

#[cfg(test)]
mod tests {
use nostr::{Event, EventBuilder, Keys, Tag};

use super::*;

fn create_test_event(content: &str) -> Event {
let keys = Keys::generate();
EventBuilder::text_note(content)
.sign_with_keys(&keys)
.unwrap()
}

#[test]
fn test_search_match_in_content() {
let event = create_test_event("Hello World");
let event: EventBorrow = (&event).into();

let mut filter = DatabaseFilter::from(Filter::new());

// Case insensitive match
filter.search = Some("hello".to_string());
assert!(filter.match_event(&event));

filter.search = Some("world".to_string());
assert!(filter.match_event(&event));

// No match
filter.search = Some("rust".to_string());
assert!(!filter.match_event(&event));
}

#[test]
fn test_search_match_in_tags() {
let keys = Keys::generate();
let event = EventBuilder::text_note("content")
.tag(Tag::parse(["title", "Search userfacing tags"]).unwrap())
.sign_with_keys(&keys)
.unwrap();
let event: EventBorrow = (&event).into();

let mut filter = DatabaseFilter::from(Filter::new());

filter.search = Some("userfacing".to_string());
assert!(filter.match_event(&event));

filter.search = Some("bitcoin".to_string());
assert!(!filter.match_event(&event));
}

#[test]
fn test_search_empty_query() {
let event = create_test_event("test");
let event: EventBorrow = (&event).into();
let mut filter = DatabaseFilter::from(Filter::new());

filter.search = Some("".to_string());
assert!(!filter.match_event(&event));
}

#[test]
fn test_search_no_query() {
let event = create_test_event("test");
let event: EventBorrow = (&event).into();
let filter = DatabaseFilter::from(Filter::new());

assert!(filter.match_event(&event));
}

#[test]
fn test_search_partial_match() {
let event = create_test_event("nostr protocol");
let event: EventBorrow = (&event).into();
let mut filter = DatabaseFilter::from(Filter::new());

filter.search = Some("proto".to_string());
assert!(filter.match_event(&event));
}
}
1 change: 1 addition & 0 deletions database/nostr-lmdb/src/store/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ mod error;
mod filter;
mod ingester;
mod lmdb;
mod query;

use self::error::Error;
use self::ingester::{Ingester, IngesterItem};
Expand Down
Loading
Loading