From 5d76cfc0f2c7733ea764305f05b4a85a2dbba72b Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 2 Nov 2025 20:24:46 +0100 Subject: [PATCH 001/136] misc: make index generic --- ixx/src/action/mod.rs | 2 +- libixx/src/index.rs | 97 +++++++++++++++++++++++-------------------- 2 files changed, 54 insertions(+), 45 deletions(-) diff --git a/ixx/src/action/mod.rs b/ixx/src/action/mod.rs index 6f76f0d..bfebf34 100644 --- a/ixx/src/action/mod.rs +++ b/ixx/src/action/mod.rs @@ -1,3 +1,3 @@ pub(crate) mod index; -pub(crate) mod search; pub(crate) mod meta; +pub(crate) mod search; diff --git a/libixx/src/index.rs b/libixx/src/index.rs index f5b4e2d..7c2ec8b 100644 --- a/libixx/src/index.rs +++ b/libixx/src/index.rs @@ -1,6 +1,6 @@ use std::io::{Cursor, Read, Seek, Write}; -use binrw::{binrw, BinRead, BinWrite, Endian, NullString}; +use binrw::{BinRead, BinWrite, Endian, NullString, binrw}; use crate::IxxError; @@ -9,10 +9,10 @@ use crate::IxxError; #[derive(Debug, Clone, PartialEq)] pub struct Index { meta: Meta, - #[bw(calc = options.len() as u32)] + #[bw(calc = entries.len() as u32)] count: u32, #[br(count = count)] - options: Vec, + entries: Vec, } #[binrw] @@ -27,7 +27,7 @@ pub struct Meta { #[binrw] #[derive(Default, Debug, Clone, PartialEq)] -pub struct OptionEntry { +pub struct Entry { /// index in the scopes Vec scope_id: u8, #[bw(calc = labels.len() as u16)] @@ -39,7 +39,7 @@ pub struct OptionEntry { #[binrw] #[derive(Debug, Clone, PartialEq)] struct Reference { - option_idx: u16, + entry_idx: u16, label_idx: u8, } @@ -59,7 +59,7 @@ impl Index { chunk_size, scopes: vec![], }, - options: vec![], + entries: vec![], } } @@ -75,21 +75,21 @@ impl Index { Ok(BinWrite::write_options(self, write, Endian::Little, ())?) } - pub fn push(&mut self, scope_id: u8, option: &str) { - let labels = option + pub fn push(&mut self, scope_id: u8, name: &str) { + let labels = name .split('.') .map(|segment| { let segment = segment.into(); - for (option_idx, OptionEntry { labels: option, .. }) in self.options.iter().enumerate() { - for (label_idx, label) in option.iter().enumerate() { + for (entry_idx, Entry { labels, .. }) in self.entries.iter().enumerate() { + for (label_idx, label) in labels.iter().enumerate() { if let Label::InPlace(inplace) = label { if inplace != &segment { continue; } return Label::Reference(Reference { - option_idx: option_idx as u16, + entry_idx: entry_idx as u16, label_idx: label_idx as u8, }); } @@ -100,17 +100,17 @@ impl Index { }) .collect(); - self.options.push(OptionEntry { scope_id, labels }); + self.entries.push(Entry { scope_id, labels }); } fn resolve_reference(&self, reference: &Reference) -> Result<&NullString, IxxError> { - let option_idx = reference.option_idx as usize; + let entry_idx = reference.entry_idx as usize; - if self.options.len() <= option_idx { + if self.entries.len() <= entry_idx { return Err(IxxError::ReferenceOutOfBounds); } - let entry = &self.options[option_idx].labels; + let entry = &self.entries[entry_idx].labels; let label_idx = reference.label_idx as usize; @@ -126,21 +126,28 @@ impl Index { } } - pub fn get_idx_by_name(&self, scope_id: u8, option: &str) -> Result, IxxError> { + pub fn get_idx_by_name(&self, scope_id: u8, name: &str) -> Result, IxxError> { let mut labels = Vec::new(); - for segment in option.split('.') { + for segment in name.split('.') { let segment = segment.into(); 'outer: { - for (option_idx, OptionEntry { labels: option, .. }) in self.options.iter().enumerate() { - for (label_idx, label) in option.iter().enumerate() { + for ( + entry_idx, + Entry { + labels: inner_labels, + .. + }, + ) in self.entries.iter().enumerate() + { + for (label_idx, label) in inner_labels.iter().enumerate() { if let Label::InPlace(inplace) = label { if inplace != &segment { continue; } labels.push(Reference { - option_idx: option_idx as u16, + entry_idx: entry_idx as u16, label_idx: label_idx as u8, }); break 'outer; @@ -154,17 +161,17 @@ impl Index { Ok( self - .options + .entries .iter() .enumerate() .find( |( idx, - OptionEntry { - scope_id: option_scope_id, - labels: option, + Entry { + scope_id: entry_scope_id, + labels: inner_labels, }, - )| *option_scope_id == scope_id && do_labels_match(*idx, option, &labels), + )| *entry_scope_id == scope_id && do_labels_match(*idx, inner_labels, &labels), ) .map(|(idx, _)| idx), ) @@ -185,45 +192,45 @@ impl Index { for ( idx, - OptionEntry { - scope_id: option_scope_id, - labels: option, + Entry { + scope_id: entry_scope_id, + labels, }, - ) in self.options.iter().enumerate() + ) in self.entries.iter().enumerate() { if let Some(scope_id) = scope_id { - if *option_scope_id != scope_id { + if *entry_scope_id != scope_id { continue; } } - let mut option_name = String::new(); - for label in option { - option_name.push_str(&String::try_from( + let mut entry_name = String::new(); + for label in labels { + entry_name.push_str(&String::try_from( match label { Label::InPlace(data) => data, Label::Reference(reference) => self.resolve_reference(reference)?, } .clone(), )?); - option_name.push('.') + entry_name.push('.') } // remove last dot... - option_name.pop(); + entry_name.pop(); - let lower_option_name = option_name.to_lowercase(); + let lower_entry_name = entry_name.to_lowercase(); let mut start = 0; 'outer: { for segment in &search { - match lower_option_name[start..].find(segment) { + match lower_entry_name[start..].find(segment) { Some(idx) => start = idx + segment.len(), None => break 'outer, } } - results.push((idx, *option_scope_id, option_name)); + results.push((idx, *entry_scope_id, entry_name)); if results.len() >= max_results { return Ok(results); } @@ -239,7 +246,9 @@ impl Index { pub fn push_scope(&mut self, scope: String) -> u8 { if self.meta.scopes.len() == u8::MAX.into() { - panic!("You reached the limit of 256 scopes. Please contact the developers for further assistance."); + panic!( + "You reached the limit of 256 scopes. Please contact the developers for further assistance." + ); } let idx = self.meta.scopes.len(); @@ -248,18 +257,18 @@ impl Index { } } -fn do_labels_match(option_idx: usize, option: &[Label], search: &[Reference]) -> bool { - let matching = option +fn do_labels_match(entry_idx: usize, labels: &[Label], search: &[Reference]) -> bool { + let matching = labels .iter() .enumerate() .zip(search.iter()) - .filter(|&((label_idx, option), search)| match option { + .filter(|&((label_idx, entry), search)| match entry { Label::InPlace(_) => { - option_idx == search.option_idx as usize && label_idx == search.label_idx as usize + entry_idx == search.entry_idx as usize && label_idx == search.label_idx as usize } Label::Reference(reference) => reference == search, }) .count(); - matching == option.len() && matching == search.len() + matching == labels.len() && matching == search.len() } From 47de101fb954dc37a62334d8c436229357e964a9 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 2 Nov 2025 23:49:37 +0100 Subject: [PATCH 002/136] packages: init --- Cargo.lock | 35 +++++ ixx/Cargo.toml | 5 + ixx/src/action/index/mod.rs | 66 ++++++++ ixx/src/action/{index.rs => index/options.rs} | 119 +++++++++----- ixx/src/action/index/packages.rs | 146 ++++++++++++++++++ ixx/src/args.rs | 16 +- ixx/src/main.rs | 6 +- ixx/src/package.rs | 25 +++ libixx/src/lib.rs | 2 + libixx/src/package.rs | 17 ++ 10 files changed, 388 insertions(+), 49 deletions(-) create mode 100644 ixx/src/action/index/mod.rs rename ixx/src/action/{index.rs => index/options.rs} (69%) create mode 100644 ixx/src/action/index/packages.rs create mode 100644 ixx/src/package.rs create mode 100644 libixx/src/package.rs diff --git a/Cargo.lock b/Cargo.lock index 4f7e075..6c4f90c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -109,6 +109,12 @@ version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c76a5792e44e4abe34d3abf15636779261d45a7450612059293d1d2cfc63422" +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + [[package]] name = "cc" version = "1.2.30" @@ -361,6 +367,7 @@ dependencies = [ "markdown", "serde", "serde_json", + "tokio", "tree-sitter-highlight", "tree-sitter-nix", "url", @@ -428,6 +435,12 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + [[package]] name = "potential_utf" version = "0.1.2" @@ -622,6 +635,28 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tokio" +version = "1.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +dependencies = [ + "bytes", + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tree-sitter" version = "0.25.9" diff --git a/ixx/Cargo.toml b/ixx/Cargo.toml index 955290b..3fefc3e 100644 --- a/ixx/Cargo.toml +++ b/ixx/Cargo.toml @@ -5,6 +5,11 @@ edition = "2024" repository = "https://github.com/NuschtOS/ixx/" license = "MIT OR Apache-2.0" +[dependencies.tokio] +version = "1.48" +default-features = false +features = ["macros", "rt-multi-thread", "fs", "io-util"] + [dependencies] serde = { version = "1.0", features = ["derive"] } clap = { version = "4.5", features = ["derive"] } diff --git a/ixx/src/action/index/mod.rs b/ixx/src/action/index/mod.rs new file mode 100644 index 0000000..d86f5cf --- /dev/null +++ b/ixx/src/action/index/mod.rs @@ -0,0 +1,66 @@ +use std::path::PathBuf; + +use anyhow::Context; +use serde::Deserialize; +use tokio::join; +use url::Url; + +use crate::{ + action::index::{options::index_options, packages::index_packages}, + args::IndexModule, +}; + +mod options; +mod packages; + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct Config { + scopes: Vec, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct Scope { + name: Option, + options_json: PathBuf, + packages_json: PathBuf, + url_prefix: Url, + options_prefix: Option, +} + +struct OptionEntry { + name: String, + scope: u8, + option: libixx::Option, +} + +struct PackageEntry { + name: String, + scope: u8, + option: libixx::Package, +} + +pub(crate) async fn index(module: IndexModule) -> anyhow::Result<()> { + let config: Config = { + let raw_config = tokio::fs::read_to_string(&module.config) + .await + .with_context(|| { + format!( + "Failed to read config file: {}", + module.config.to_string_lossy() + ) + })?; + serde_json::from_str(&raw_config)? + }; + + let (options_result, packages_result) = join!( + index_options(&module, &config), + index_packages(&module, &config), + ); + + options_result?; + packages_result?; + + Ok(()) +} diff --git a/ixx/src/action/index.rs b/ixx/src/action/index/options.rs similarity index 69% rename from ixx/src/action/index.rs rename to ixx/src/action/index/options.rs index e52bdb9..d14c5f0 100644 --- a/ixx/src/action/index.rs +++ b/ixx/src/action/index/options.rs @@ -1,52 +1,39 @@ -use std::{collections::HashMap, fs::File, path::PathBuf}; +use std::{collections::HashMap, io::Cursor}; -use anyhow::anyhow; +use anyhow::{Context, anyhow}; use libixx::Index; -use serde::Deserialize; +use tokio::{fs::File, io::AsyncWriteExt, task::JoinSet}; use url::Url; use crate::{ + action::index::{Config, OptionEntry}, args::IndexModule, option::{self, Declaration}, }; -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct Config { - scopes: Vec, -} - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct Scope { - name: Option, - options_json: PathBuf, - url_prefix: Url, - options_prefix: Option, -} - -struct Entry { - name: String, - scope: u8, - option: libixx::Option, -} - -pub(crate) fn index(module: IndexModule) -> anyhow::Result<()> { - let mut raw_options: Vec = vec![]; - - let config_file = File::open(module.config)?; - let config: Config = serde_json::from_reader(config_file)?; +pub(crate) async fn index_options(module: &IndexModule, config: &Config) -> anyhow::Result<()> { + let mut raw_options: Vec = vec![]; let mut index = Index::new(module.chunk_size); - for scope in config.scopes { + for scope in &config.scopes { println!("Parsing {}", scope.options_json.to_string_lossy()); - let file = File::open(scope.options_json)?; - let options: HashMap = serde_json::from_reader(file)?; + let options: HashMap = { + let raw_options = tokio::fs::read_to_string(&scope.options_json) + .await + .with_context(|| { + format!( + "Failed to read options json: {}", + scope.options_json.to_string_lossy() + ) + })?; + serde_json::from_str(&raw_options)? + }; let scope_idx = index.push_scope( scope .name + .as_ref() .map(|x| x.to_string()) .unwrap_or_else(|| scope.url_prefix.to_string()), ); @@ -67,8 +54,10 @@ pub(crate) fn index(module: IndexModule) -> anyhow::Result<()> { Some(prefix) => format!("{}.{}", prefix, name), None => name, }; + let option = into_option(&scope.url_prefix, &name, option)?; - raw_options.push(Entry { + + raw_options.push(OptionEntry { name, scope: scope_idx, option, @@ -82,19 +71,47 @@ pub(crate) fn index(module: IndexModule) -> anyhow::Result<()> { println!("Sorted options"); + println!("Building options index..."); for entry in &raw_options { index.push(entry.scope, &entry.name); } - println!("Writing index to {}", module.index_output.to_string_lossy()); + println!( + "Writing options index to {}", + module.options_index_output.to_string_lossy() + ); + + { + let index_buf = { + let mut buf = Vec::new(); + index.write_into(&mut Cursor::new(&mut buf))?; + buf + }; + + let mut index_output = File::create(&module.options_index_output) + .await + .with_context(|| { + format!( + "Failed to create {}", + module.options_index_output.to_string_lossy() + ) + })?; - let mut output = File::create(module.index_output)?; - index.write_into(&mut output)?; + index_output.write_all(index_buf.as_slice()).await?; + } - println!("Writing meta to {}", module.meta_output.to_string_lossy()); + println!( + "Writing meta to {}", + module.options_meta_output.to_string_lossy() + ); - if !module.meta_output.exists() { - std::fs::create_dir(&module.meta_output)?; + if !module.options_meta_output.exists() { + std::fs::create_dir(&module.options_meta_output).with_context(|| { + format!( + "Failed to create dir {}", + module.options_meta_output.to_string_lossy() + ) + })?; } let options = raw_options @@ -102,9 +119,27 @@ pub(crate) fn index(module: IndexModule) -> anyhow::Result<()> { .map(|entry| entry.option) .collect::>(); + let mut join_set = JoinSet::new(); + for (idx, chunk) in options.chunks(module.chunk_size as usize).enumerate() { - let mut file = File::create(module.meta_output.join(format!("{}.json", idx)))?; - serde_json::to_writer(&mut file, chunk)?; + let path = module.options_meta_output.join(format!("{}.json", idx)); + + let meta_string = serde_json::to_string(chunk) + .with_context(|| format!("Failed to write to {}", path.to_string_lossy()))?; + + join_set.spawn(async move { + let mut file = File::create(&path) + .await + .with_context(|| format!("Failed to create {}", path.to_string_lossy()))?; + + file.write_all(meta_string.as_bytes()).await?; + + Ok::<_, anyhow::Error>(()) + }); + } + + while let Some(result) = join_set.join_next().await { + result??; } Ok(()) @@ -171,7 +206,7 @@ fn update_declaration(url_prefix: &Url, declaration: Declaration) -> anyhow::Res mod test { use url::Url; - use crate::{action::index::update_declaration, option::Declaration}; + use crate::{action::index::options::update_declaration, option::Declaration}; #[test] fn test_update_declaration() { diff --git a/ixx/src/action/index/packages.rs b/ixx/src/action/index/packages.rs new file mode 100644 index 0000000..4ce514c --- /dev/null +++ b/ixx/src/action/index/packages.rs @@ -0,0 +1,146 @@ +use std::io::Cursor; + +use anyhow::Context; +use libixx::Index; +use tokio::{fs::File, io::AsyncWriteExt, task::JoinSet}; + +use crate::{ + action::index::{Config, PackageEntry}, + args::IndexModule, + package::{self, OneOrMany}, +}; + +pub(crate) async fn index_packages(module: &IndexModule, config: &Config) -> anyhow::Result<()> { + let mut raw_packages: Vec = vec![]; + + let mut index = Index::new(module.chunk_size); + + for scope in &config.scopes { + let scope_idx = index.push_scope( + scope + .name + .as_ref() + .map(|x| x.to_string()) + .unwrap_or_else(|| scope.url_prefix.to_string()), + ); + + println!("Parsing {}", scope.packages_json.to_string_lossy()); + let packages: Vec = { + let raw_packages = tokio::fs::read_to_string(&scope.packages_json) + .await + .with_context(|| { + format!( + "Failed to read options json: {}", + scope.options_json.to_string_lossy() + ) + })?; + serde_json::from_str(&raw_packages)? + }; + + for package in packages { + raw_packages.push(PackageEntry { + name: package.attr_name.clone(), + scope: scope_idx, + option: into_package(package)?, + }); + } + } + + println!("Read {} packages", raw_packages.len()); + + raw_packages.sort_by(|a, b| a.name.cmp(&b.name)); + + println!("Sorted packages"); + + for entry in &raw_packages { + index.push(entry.scope, &entry.name); + } + + println!( + "Writing packages index to {}", + module.packages_index_output.to_string_lossy() + ); + + { + let index_buf = { + let mut buf = Vec::new(); + index.write_into(&mut Cursor::new(&mut buf))?; + buf + }; + + let mut index_output = File::create(&module.packages_index_output) + .await + .with_context(|| { + format!( + "Failed to create {}", + module.packages_index_output.to_string_lossy() + ) + })?; + + index_output.write_all(index_buf.as_slice()).await?; + } + + println!( + "write meta to {}", + module.packages_meta_output.to_string_lossy() + ); + + if !module.packages_meta_output.exists() { + std::fs::create_dir(&module.packages_meta_output).with_context(|| { + format!( + "Failed to create dir {}", + module.packages_meta_output.to_string_lossy() + ) + })?; + } + + let packages = raw_packages + .into_iter() + .map(|entry| entry.option) + .collect::>(); + + let mut join_set = JoinSet::new(); + + for (idx, chunk) in packages.chunks(module.chunk_size as usize).enumerate() { + let path = module.packages_meta_output.join(format!("{}.json", idx)); + + let meta_string = serde_json::to_string(chunk) + .with_context(|| format!("Failed to write to {}", path.to_string_lossy()))?; + + join_set.spawn(async move { + let mut file = File::create(&path) + .await + .with_context(|| format!("Failed to create {}", path.to_string_lossy()))?; + + file.write_all(meta_string.as_bytes()).await?; + + Ok::<_, anyhow::Error>(()) + }); + } + + while let Some(result) = join_set.join_next().await { + result??; + } + + Ok(()) +} + +fn into_package(package: package::Package) -> anyhow::Result { + Ok(libixx::Package { + attr_name: package.attr_name, + eval_error: package.eval_error, + broken: package.broken, + description: package.description, + homepages: match package.homepage { + None => vec![], + Some(OneOrMany::One(homepage)) => vec![homepage], + Some(OneOrMany::Many(homepages)) => homepages, + }, + outputs: package.outputs.unwrap_or_else(|| vec![]), + insecure: package.insecure, + name: package.name, + pname: package.pname, + unfree: package.unfree, + version: package.version, + }) +} diff --git a/ixx/src/args.rs b/ixx/src/args.rs index fcfea03..766dc3e 100644 --- a/ixx/src/args.rs +++ b/ixx/src/args.rs @@ -24,13 +24,19 @@ pub(super) enum Format { pub(super) struct IndexModule { pub(super) config: PathBuf, - #[clap(short, long, default_value = "index.ixx")] - pub(super) index_output: PathBuf, + #[clap(long, default_value = "options/index.ixx")] + pub(super) options_index_output: PathBuf, + + #[clap(long, default_value = "options/meta")] + pub(crate) options_meta_output: PathBuf, + + #[clap(long, default_value = "packages/index.ixx")] + pub(super) packages_index_output: PathBuf, - #[clap(short, long, default_value = "meta")] - pub(crate) meta_output: PathBuf, + #[clap(long, default_value = "packages/meta")] + pub(crate) packages_meta_output: PathBuf, - #[clap(short, long, default_value = "100")] + #[clap(long, default_value = "100")] pub(super) chunk_size: u32, } diff --git a/ixx/src/main.rs b/ixx/src/main.rs index e6d57c0..547fb31 100644 --- a/ixx/src/main.rs +++ b/ixx/src/main.rs @@ -4,13 +4,15 @@ use clap::Parser; mod action; mod args; mod option; +mod package; pub(crate) mod utils; -fn main() -> anyhow::Result<()> { +#[tokio::main] +async fn main() -> anyhow::Result<()> { let args = Args::parse(); match args.action { - Action::Index(module) => action::index::index(module), + Action::Index(module) => action::index::index(module).await, Action::Search(module) => action::search::search(module), Action::Meta(module) => action::meta::meta(module), }?; diff --git a/ixx/src/package.rs b/ixx/src/package.rs new file mode 100644 index 0000000..c72400d --- /dev/null +++ b/ixx/src/package.rs @@ -0,0 +1,25 @@ +use serde::Deserialize; +use url::Url; + +#[derive(Debug, Clone, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Package { + pub attr_name: String, + pub eval_error: Option, + pub broken: Option, + pub description: Option, + pub homepage: Option>, + pub outputs: Option>, + pub insecure: Option, + pub name: Option, + pub pname: Option, + pub unfree: Option, + pub version: Option, +} + +#[derive(Debug, Clone, PartialEq, Deserialize)] +#[serde(untagged)] +pub enum OneOrMany { + One(T), + Many(Vec), +} diff --git a/libixx/src/lib.rs b/libixx/src/lib.rs index 841e026..c151d7b 100644 --- a/libixx/src/lib.rs +++ b/libixx/src/lib.rs @@ -1,10 +1,12 @@ pub use error::IxxError; pub use index::Index; pub use option::Option; +pub use package::Package; mod error; mod index; mod option; +mod package; #[cfg(test)] mod test; diff --git a/libixx/src/package.rs b/libixx/src/package.rs new file mode 100644 index 0000000..09c5d89 --- /dev/null +++ b/libixx/src/package.rs @@ -0,0 +1,17 @@ +use serde::{Deserialize, Serialize}; +use url::Url; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Package { + pub attr_name: String, + pub eval_error: Option, + pub broken: Option, + pub description: Option, + pub homepages: Vec, + pub outputs: Vec, + pub insecure: Option, + pub name: Option, + pub pname: Option, + pub unfree: Option, + pub version: Option, +} From cf805a4e76c041ef38c35cc9c568e8635546e082 Mon Sep 17 00:00:00 2001 From: Marcel Date: Mon, 3 Nov 2025 22:17:30 +0100 Subject: [PATCH 003/136] index: disable compression if there is not dot in name --- libixx/src/index.rs | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/libixx/src/index.rs b/libixx/src/index.rs index 7c2ec8b..8c411d4 100644 --- a/libixx/src/index.rs +++ b/libixx/src/index.rs @@ -76,29 +76,34 @@ impl Index { } pub fn push(&mut self, scope_id: u8, name: &str) { - let labels = name - .split('.') - .map(|segment| { - let segment = segment.into(); - - for (entry_idx, Entry { labels, .. }) in self.entries.iter().enumerate() { - for (label_idx, label) in labels.iter().enumerate() { - if let Label::InPlace(inplace) = label { - if inplace != &segment { - continue; + // optimize, if there is no dot in the name, compression does not make sense + let labels = if !name.contains('.') { + vec![Label::InPlace(name.into())] + } else { + name + .split('.') + .map(|segment| { + let segment = segment.into(); + + for (entry_idx, Entry { labels, .. }) in self.entries.iter().enumerate() { + for (label_idx, label) in labels.iter().enumerate() { + if let Label::InPlace(inplace) = label { + if inplace != &segment { + continue; + } + + return Label::Reference(Reference { + entry_idx: entry_idx as u16, + label_idx: label_idx as u8, + }); } - - return Label::Reference(Reference { - entry_idx: entry_idx as u16, - label_idx: label_idx as u8, - }); } } - } - Label::InPlace(segment) - }) - .collect(); + Label::InPlace(segment) + }) + .collect() + }; self.entries.push(Entry { scope_id, labels }); } From cda4a88c63a34a42f978668dc6e4261300ba1c9e Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 5 Nov 2025 11:19:17 +0100 Subject: [PATCH 004/136] ixx: allow list of packages json files --- ixx/src/action/index/mod.rs | 2 +- ixx/src/action/index/packages.rs | 52 +++++++++++++++++++++----------- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/ixx/src/action/index/mod.rs b/ixx/src/action/index/mod.rs index d86f5cf..b6f99fe 100644 --- a/ixx/src/action/index/mod.rs +++ b/ixx/src/action/index/mod.rs @@ -24,7 +24,7 @@ pub(crate) struct Config { pub(crate) struct Scope { name: Option, options_json: PathBuf, - packages_json: PathBuf, + packages_jsons: Vec, url_prefix: Url, options_prefix: Option, } diff --git a/ixx/src/action/index/packages.rs b/ixx/src/action/index/packages.rs index 4ce514c..c50b831 100644 --- a/ixx/src/action/index/packages.rs +++ b/ixx/src/action/index/packages.rs @@ -24,25 +24,41 @@ pub(crate) async fn index_packages(module: &IndexModule, config: &Config) -> any .unwrap_or_else(|| scope.url_prefix.to_string()), ); - println!("Parsing {}", scope.packages_json.to_string_lossy()); - let packages: Vec = { - let raw_packages = tokio::fs::read_to_string(&scope.packages_json) - .await - .with_context(|| { - format!( - "Failed to read options json: {}", - scope.options_json.to_string_lossy() - ) - })?; - serde_json::from_str(&raw_packages)? - }; - - for package in packages { - raw_packages.push(PackageEntry { - name: package.attr_name.clone(), - scope: scope_idx, - option: into_package(package)?, + let mut join_set = JoinSet::new(); + + for packages_json in &scope.packages_jsons { + let packages_json = packages_json.clone(); + join_set.spawn(async move { + println!("Parsing {}", packages_json.to_string_lossy()); + let packages: Vec = { + let raw_packages = tokio::fs::read_to_string(&packages_json) + .await + .with_context(|| { + format!( + "Failed to read options json: {}", + packages_json.to_string_lossy() + ) + })?; + serde_json::from_str(&raw_packages)? + }; + + let packages = packages + .into_iter() + .map(|package| { + Ok::<_, anyhow::Error>(PackageEntry { + name: package.attr_name.clone(), + scope: scope_idx, + option: into_package(package)?, + }) + }) + .collect::, _>>()?; + + Ok::<_, anyhow::Error>(packages) }); + + while let Some(result) = join_set.join_next().await { + raw_packages.extend(result??); + } } } From 0ac51146f1d1e9f6d5276fb6f0707c1305514a53 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 5 Nov 2025 16:20:26 +0100 Subject: [PATCH 005/136] index: add reference idx safeguard --- libixx/src/index.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libixx/src/index.rs b/libixx/src/index.rs index 8c411d4..f3a21d3 100644 --- a/libixx/src/index.rs +++ b/libixx/src/index.rs @@ -151,6 +151,12 @@ impl Index { continue; } + if entry_idx >= u16::MAX.into() { + panic!( + "You can not reference names after index 65535. Please contact the developers for further assistance." + ); + } + labels.push(Reference { entry_idx: entry_idx as u16, label_idx: label_idx as u8, From d1a81f04dc8295f303a941e6db89a0abb90ec7d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Thu, 6 Nov 2025 00:26:55 +0100 Subject: [PATCH 006/136] Allow passing either options or packages --- ixx/src/action/index/mod.rs | 8 ++++---- ixx/src/action/index/options.rs | 16 ++++++++++++---- ixx/src/action/index/packages.rs | 12 ++++++++++-- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/ixx/src/action/index/mod.rs b/ixx/src/action/index/mod.rs index b6f99fe..15e6f2c 100644 --- a/ixx/src/action/index/mod.rs +++ b/ixx/src/action/index/mod.rs @@ -13,18 +13,18 @@ use crate::{ mod options; mod packages; -#[derive(Deserialize)] +#[derive(Debug,Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct Config { scopes: Vec, } -#[derive(Deserialize)] +#[derive(Debug,Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct Scope { name: Option, - options_json: PathBuf, - packages_jsons: Vec, + options_json: Option, + packages_jsons: Option>, url_prefix: Url, options_prefix: Option, } diff --git a/ixx/src/action/index/options.rs b/ixx/src/action/index/options.rs index d14c5f0..8f982f9 100644 --- a/ixx/src/action/index/options.rs +++ b/ixx/src/action/index/options.rs @@ -17,14 +17,19 @@ pub(crate) async fn index_options(module: &IndexModule, config: &Config) -> anyh let mut index = Index::new(module.chunk_size); for scope in &config.scopes { - println!("Parsing {}", scope.options_json.to_string_lossy()); + let options_json = match &scope.options_json { + Some(packages_jsons) => packages_jsons, + None => { continue; } + }; + + println!("Parsing {}", options_json.to_string_lossy()); let options: HashMap = { - let raw_options = tokio::fs::read_to_string(&scope.options_json) + let raw_options = tokio::fs::read_to_string(&options_json) .await .with_context(|| { format!( "Failed to read options json: {}", - scope.options_json.to_string_lossy() + options_json.to_string_lossy() ) })?; serde_json::from_str(&raw_options)? @@ -66,6 +71,9 @@ pub(crate) async fn index_options(module: &IndexModule, config: &Config) -> anyh } println!("Read {} options", raw_options.len()); + if raw_options.len() == 0 { + return Ok(()); + } raw_options.sort_by(|a, b| a.name.cmp(&b.name)); @@ -101,7 +109,7 @@ pub(crate) async fn index_options(module: &IndexModule, config: &Config) -> anyh } println!( - "Writing meta to {}", + "Writing options meta to {}", module.options_meta_output.to_string_lossy() ); diff --git a/ixx/src/action/index/packages.rs b/ixx/src/action/index/packages.rs index c50b831..acda3ca 100644 --- a/ixx/src/action/index/packages.rs +++ b/ixx/src/action/index/packages.rs @@ -16,6 +16,11 @@ pub(crate) async fn index_packages(module: &IndexModule, config: &Config) -> any let mut index = Index::new(module.chunk_size); for scope in &config.scopes { + let packages_jsons = match &scope.packages_jsons { + Some(packages_jsons) => packages_jsons, + None => { continue; } + }; + let scope_idx = index.push_scope( scope .name @@ -26,7 +31,7 @@ pub(crate) async fn index_packages(module: &IndexModule, config: &Config) -> any let mut join_set = JoinSet::new(); - for packages_json in &scope.packages_jsons { + for packages_json in packages_jsons { let packages_json = packages_json.clone(); join_set.spawn(async move { println!("Parsing {}", packages_json.to_string_lossy()); @@ -63,6 +68,9 @@ pub(crate) async fn index_packages(module: &IndexModule, config: &Config) -> any } println!("Read {} packages", raw_packages.len()); + if raw_packages.len() == 0 { + return Ok(()); + } raw_packages.sort_by(|a, b| a.name.cmp(&b.name)); @@ -97,7 +105,7 @@ pub(crate) async fn index_packages(module: &IndexModule, config: &Config) -> any } println!( - "write meta to {}", + "write packages meta to {}", module.packages_meta_output.to_string_lossy() ); From 0e7daa6d3d228fdc46e0b789dcfcf3c47499c831 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Thu, 6 Nov 2025 00:57:40 +0100 Subject: [PATCH 007/136] Clippy --- ixx/src/action/index/options.rs | 4 ++-- ixx/src/action/index/packages.rs | 4 ++-- libixx/src/index.rs | 5 ++--- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ixx/src/action/index/options.rs b/ixx/src/action/index/options.rs index 8f982f9..37e36c4 100644 --- a/ixx/src/action/index/options.rs +++ b/ixx/src/action/index/options.rs @@ -71,7 +71,7 @@ pub(crate) async fn index_options(module: &IndexModule, config: &Config) -> anyh } println!("Read {} options", raw_options.len()); - if raw_options.len() == 0 { + if raw_options.is_empty() { return Ok(()); } @@ -201,7 +201,7 @@ fn update_declaration(url_prefix: &Url, declaration: Declaration) -> anyhow::Res "{}/default.nix", url .path_segments() - .map(|segments| segments.last().unwrap_or("")) + .map(|mut segments| segments.next_back().unwrap_or("")) .unwrap_or(""), ))?; } diff --git a/ixx/src/action/index/packages.rs b/ixx/src/action/index/packages.rs index acda3ca..69183e7 100644 --- a/ixx/src/action/index/packages.rs +++ b/ixx/src/action/index/packages.rs @@ -68,7 +68,7 @@ pub(crate) async fn index_packages(module: &IndexModule, config: &Config) -> any } println!("Read {} packages", raw_packages.len()); - if raw_packages.len() == 0 { + if raw_packages.is_empty() { return Ok(()); } @@ -160,7 +160,7 @@ fn into_package(package: package::Package) -> anyhow::Result { Some(OneOrMany::One(homepage)) => vec![homepage], Some(OneOrMany::Many(homepages)) => homepages, }, - outputs: package.outputs.unwrap_or_else(|| vec![]), + outputs: package.outputs.unwrap_or_default(), insecure: package.insecure, name: package.name, pname: package.pname, diff --git a/libixx/src/index.rs b/libixx/src/index.rs index f3a21d3..d932417 100644 --- a/libixx/src/index.rs +++ b/libixx/src/index.rs @@ -209,11 +209,10 @@ impl Index { }, ) in self.entries.iter().enumerate() { - if let Some(scope_id) = scope_id { - if *entry_scope_id != scope_id { + if let Some(scope_id) = scope_id + && *entry_scope_id != scope_id { continue; } - } let mut entry_name = String::new(); for label in labels { From e07f0944d83d9ef30821cfd1e2eecd245b531420 Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 6 Nov 2025 10:02:00 +0100 Subject: [PATCH 008/136] index: use HashMap for lookup while inserting --- ixx/src/action/index/mod.rs | 4 +- ixx/src/action/index/options.rs | 13 ++-- ixx/src/action/index/packages.rs | 13 ++-- libixx/src/index.rs | 108 +++++++++++++++++++------------ libixx/src/lib.rs | 2 +- libixx/src/test/mod.rs | 26 ++++---- 6 files changed, 100 insertions(+), 66 deletions(-) diff --git a/ixx/src/action/index/mod.rs b/ixx/src/action/index/mod.rs index 15e6f2c..e81ee2d 100644 --- a/ixx/src/action/index/mod.rs +++ b/ixx/src/action/index/mod.rs @@ -13,13 +13,13 @@ use crate::{ mod options; mod packages; -#[derive(Debug,Deserialize)] +#[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct Config { scopes: Vec, } -#[derive(Debug,Deserialize)] +#[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct Scope { name: Option, diff --git a/ixx/src/action/index/options.rs b/ixx/src/action/index/options.rs index 37e36c4..13d76fd 100644 --- a/ixx/src/action/index/options.rs +++ b/ixx/src/action/index/options.rs @@ -1,7 +1,7 @@ use std::{collections::HashMap, io::Cursor}; use anyhow::{Context, anyhow}; -use libixx::Index; +use libixx::{Index, IndexBuilder}; use tokio::{fs::File, io::AsyncWriteExt, task::JoinSet}; use url::Url; @@ -14,12 +14,14 @@ use crate::{ pub(crate) async fn index_options(module: &IndexModule, config: &Config) -> anyhow::Result<()> { let mut raw_options: Vec = vec![]; - let mut index = Index::new(module.chunk_size); + let mut index_builder = IndexBuilder::new(module.chunk_size); for scope in &config.scopes { let options_json = match &scope.options_json { Some(packages_jsons) => packages_jsons, - None => { continue; } + None => { + continue; + } }; println!("Parsing {}", options_json.to_string_lossy()); @@ -35,7 +37,7 @@ pub(crate) async fn index_options(module: &IndexModule, config: &Config) -> anyh serde_json::from_str(&raw_options)? }; - let scope_idx = index.push_scope( + let scope_idx = index_builder.push_scope( scope .name .as_ref() @@ -81,7 +83,7 @@ pub(crate) async fn index_options(module: &IndexModule, config: &Config) -> anyh println!("Building options index..."); for entry in &raw_options { - index.push(entry.scope, &entry.name); + index_builder.push(entry.scope, &entry.name); } println!( @@ -92,6 +94,7 @@ pub(crate) async fn index_options(module: &IndexModule, config: &Config) -> anyh { let index_buf = { let mut buf = Vec::new(); + let index: Index = index_builder.into(); index.write_into(&mut Cursor::new(&mut buf))?; buf }; diff --git a/ixx/src/action/index/packages.rs b/ixx/src/action/index/packages.rs index 69183e7..0d4a2e9 100644 --- a/ixx/src/action/index/packages.rs +++ b/ixx/src/action/index/packages.rs @@ -1,7 +1,7 @@ use std::io::Cursor; use anyhow::Context; -use libixx::Index; +use libixx::{Index, IndexBuilder}; use tokio::{fs::File, io::AsyncWriteExt, task::JoinSet}; use crate::{ @@ -13,15 +13,17 @@ use crate::{ pub(crate) async fn index_packages(module: &IndexModule, config: &Config) -> anyhow::Result<()> { let mut raw_packages: Vec = vec![]; - let mut index = Index::new(module.chunk_size); + let mut index_builder = IndexBuilder::new(module.chunk_size); for scope in &config.scopes { let packages_jsons = match &scope.packages_jsons { Some(packages_jsons) => packages_jsons, - None => { continue; } + None => { + continue; + } }; - let scope_idx = index.push_scope( + let scope_idx = index_builder.push_scope( scope .name .as_ref() @@ -77,7 +79,7 @@ pub(crate) async fn index_packages(module: &IndexModule, config: &Config) -> any println!("Sorted packages"); for entry in &raw_packages { - index.push(entry.scope, &entry.name); + index_builder.push(entry.scope, &entry.name); } println!( @@ -88,6 +90,7 @@ pub(crate) async fn index_packages(module: &IndexModule, config: &Config) -> any { let index_buf = { let mut buf = Vec::new(); + let index: Index = index_builder.into(); index.write_into(&mut Cursor::new(&mut buf))?; buf }; diff --git a/libixx/src/index.rs b/libixx/src/index.rs index d932417..7340ff4 100644 --- a/libixx/src/index.rs +++ b/libixx/src/index.rs @@ -1,9 +1,17 @@ -use std::io::{Cursor, Read, Seek, Write}; +use std::{ + collections::HashMap, + io::{Cursor, Read, Seek, Write}, +}; use binrw::{BinRead, BinWrite, Endian, NullString, binrw}; use crate::IxxError; +pub struct IndexBuilder { + index: Index, + label_cache: HashMap, u16>, +} + #[binrw] #[brw(magic = b"ixx01")] #[derive(Debug, Clone, PartialEq)] @@ -52,29 +60,20 @@ enum Label { Reference(Reference), } -impl Index { +impl IndexBuilder { pub fn new(chunk_size: u32) -> Self { Self { - meta: Meta { - chunk_size, - scopes: vec![], + index: Index { + meta: Meta { + chunk_size, + scopes: vec![], + }, + entries: vec![], }, - entries: vec![], + label_cache: HashMap::new(), } } - pub fn read(buf: &[u8]) -> Result { - Self::read_from(&mut Cursor::new(buf)) - } - - pub fn read_from(read: &mut R) -> Result { - Ok(BinRead::read_options(read, Endian::Little, ())?) - } - - pub fn write_into(&self, write: &mut W) -> Result<(), IxxError> { - Ok(BinWrite::write_options(self, write, Endian::Little, ())?) - } - pub fn push(&mut self, scope_id: u8, name: &str) { // optimize, if there is no dot in the name, compression does not make sense let labels = if !name.contains('.') { @@ -83,9 +82,11 @@ impl Index { name .split('.') .map(|segment| { - let segment = segment.into(); + let segment: NullString = segment.into(); + + if let Some(entry_idx) = self.label_cache.get(segment.as_slice()) { + let Entry { labels, .. } = &self.index.entries[*entry_idx as usize]; - for (entry_idx, Entry { labels, .. }) in self.entries.iter().enumerate() { for (label_idx, label) in labels.iter().enumerate() { if let Label::InPlace(inplace) = label { if inplace != &segment { @@ -93,19 +94,55 @@ impl Index { } return Label::Reference(Reference { - entry_idx: entry_idx as u16, + entry_idx: *entry_idx, label_idx: label_idx as u8, }); } } } + if self.index.entries.len() == u16::MAX.into() { + panic!( + "You can not have more than 65535 entries. Please contact the developers for further assistance." + ); + } + + self + .label_cache + .insert(segment.to_vec(), self.index.entries.len() as u16); + Label::InPlace(segment) }) .collect() }; - self.entries.push(Entry { scope_id, labels }); + self.index.entries.push(Entry { scope_id, labels }); + } + + pub fn push_scope(&mut self, scope: String) -> u8 { + if self.index.meta.scopes.len() == u8::MAX.into() { + panic!( + "You reached the limit of 256 scopes. Please contact the developers for further assistance." + ); + } + + let idx = self.index.meta.scopes.len(); + self.index.meta.scopes.push(scope.into()); + idx as u8 + } +} + +impl Index { + pub fn read(buf: &[u8]) -> Result { + Self::read_from(&mut Cursor::new(buf)) + } + + pub fn read_from(read: &mut R) -> Result { + Ok(BinRead::read_options(read, Endian::Little, ())?) + } + + pub fn write_into(&self, write: &mut W) -> Result<(), IxxError> { + Ok(BinWrite::write_options(self, write, Endian::Little, ())?) } fn resolve_reference(&self, reference: &Reference) -> Result<&NullString, IxxError> { @@ -151,12 +188,6 @@ impl Index { continue; } - if entry_idx >= u16::MAX.into() { - panic!( - "You can not reference names after index 65535. Please contact the developers for further assistance." - ); - } - labels.push(Reference { entry_idx: entry_idx as u16, label_idx: label_idx as u8, @@ -210,9 +241,10 @@ impl Index { ) in self.entries.iter().enumerate() { if let Some(scope_id) = scope_id - && *entry_scope_id != scope_id { - continue; - } + && *entry_scope_id != scope_id + { + continue; + } let mut entry_name = String::new(); for label in labels { @@ -253,17 +285,11 @@ impl Index { pub fn meta(&self) -> &Meta { &self.meta } +} - pub fn push_scope(&mut self, scope: String) -> u8 { - if self.meta.scopes.len() == u8::MAX.into() { - panic!( - "You reached the limit of 256 scopes. Please contact the developers for further assistance." - ); - } - - let idx = self.meta.scopes.len(); - self.meta.scopes.push(scope.into()); - idx as u8 +impl From for Index { + fn from(value: IndexBuilder) -> Self { + value.index } } diff --git a/libixx/src/lib.rs b/libixx/src/lib.rs index c151d7b..2c78821 100644 --- a/libixx/src/lib.rs +++ b/libixx/src/lib.rs @@ -1,5 +1,5 @@ pub use error::IxxError; -pub use index::Index; +pub use index::{Index, IndexBuilder}; pub use option::Option; pub use package::Package; diff --git a/libixx/src/test/mod.rs b/libixx/src/test/mod.rs index 167769f..ff91ec9 100644 --- a/libixx/src/test/mod.rs +++ b/libixx/src/test/mod.rs @@ -1,20 +1,22 @@ -use crate::Index; +use crate::{Index, IndexBuilder}; #[test] fn test() { // chunk_size does not really matter here currently - let mut index = Index::new(100); + let mut index_builder = IndexBuilder::new(100); - index.push(0, "home.enableDebugInfo"); - index.push(0, "home.enableNixpkgsReleaseCheck"); - index.push(0, "home.file..enable"); - index.push(0, "home.language.measurement"); - index.push(0, "home.pointerCursor.gtk.enable"); - index.push(0, "home.pointerCursor.x11.enable"); - index.push(0, "programs.home-manager.enable"); - index.push(0, "services.home-manager.autoUpgrade.enable"); - index.push(0, "services.home-manager.autoUpgrade.frequency"); - index.push(1, "home.enableDebugInfo"); + index_builder.push(0, "home.enableDebugInfo"); + index_builder.push(0, "home.enableNixpkgsReleaseCheck"); + index_builder.push(0, "home.file..enable"); + index_builder.push(0, "home.language.measurement"); + index_builder.push(0, "home.pointerCursor.gtk.enable"); + index_builder.push(0, "home.pointerCursor.x11.enable"); + index_builder.push(0, "programs.home-manager.enable"); + index_builder.push(0, "services.home-manager.autoUpgrade.enable"); + index_builder.push(0, "services.home-manager.autoUpgrade.frequency"); + index_builder.push(1, "home.enableDebugInfo"); + + let index: Index = index_builder.into(); assert_eq!( index.search(None, "ho*auto", 10).unwrap(), From 8a43680a324eff57cd4979c7ad9b304f70e3859d Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 6 Nov 2025 11:35:28 +0100 Subject: [PATCH 009/136] index: also cache label index --- libixx/src/index.rs | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/libixx/src/index.rs b/libixx/src/index.rs index 7340ff4..d93a13f 100644 --- a/libixx/src/index.rs +++ b/libixx/src/index.rs @@ -9,7 +9,7 @@ use crate::IxxError; pub struct IndexBuilder { index: Index, - label_cache: HashMap, u16>, + label_cache: HashMap, (u16, u8)>, } #[binrw] @@ -81,24 +81,15 @@ impl IndexBuilder { } else { name .split('.') - .map(|segment| { + .enumerate() + .map(|(label_idx, segment)| { let segment: NullString = segment.into(); - if let Some(entry_idx) = self.label_cache.get(segment.as_slice()) { - let Entry { labels, .. } = &self.index.entries[*entry_idx as usize]; - - for (label_idx, label) in labels.iter().enumerate() { - if let Label::InPlace(inplace) = label { - if inplace != &segment { - continue; - } - - return Label::Reference(Reference { - entry_idx: *entry_idx, - label_idx: label_idx as u8, - }); - } - } + if let Some((entry_idx, label_idx)) = self.label_cache.get(segment.as_slice()) { + return Label::Reference(Reference { + entry_idx: *entry_idx, + label_idx: *label_idx, + }); } if self.index.entries.len() == u16::MAX.into() { @@ -109,7 +100,7 @@ impl IndexBuilder { self .label_cache - .insert(segment.to_vec(), self.index.entries.len() as u16); + .insert(segment.to_vec(), (self.index.entries.len() as u16, label_idx as u8)); Label::InPlace(segment) }) From d0061f57f39862d974bd4c0c1df27ba2781a8b8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Fri, 7 Nov 2025 00:02:38 +0100 Subject: [PATCH 010/136] Parse meta.position --- ixx/src/action/index/options.rs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/ixx/src/action/index/options.rs b/ixx/src/action/index/options.rs index 13d76fd..542aac0 100644 --- a/ixx/src/action/index/options.rs +++ b/ixx/src/action/index/options.rs @@ -179,6 +179,7 @@ fn into_option( fn update_declaration(url_prefix: &Url, declaration: Declaration) -> anyhow::Result { let mut url = match declaration { Declaration::StorePath(path) => { + let mut url_path; if path.starts_with("/") { let idx = path .match_indices('/') @@ -188,10 +189,16 @@ fn update_declaration(url_prefix: &Url, declaration: Declaration) -> anyhow::Res // +1 to also remove the / itself, when we join it with a url, the path in the url would // get removed if we won't remove it. + 1; - url_prefix.join(path.split_at(idx).1)? + url_path = path.split_at(idx).1.to_owned(); } else { - url_prefix.join(&path)? + url_path = path } + + if let Some((path, line)) = url_path.split_once(':') { + url_path = format!("{path}#L{line}"); + } + + url_prefix.join(&url_path)? } Declaration::Url { name: _, url } => url, }; @@ -254,6 +261,18 @@ mod test { Url::parse("https://example.com/some/path/modules/initrd.nix").unwrap() ); + // package position + assert_eq!( + update_declaration( + &Url::parse("https://example.com/some/path/").unwrap(), + Declaration::StorePath( + "/nix/store/pb93n2bk2zpyn1sqpkm3gyhra26zy4ps-source/pkgs/by-name/he/hello/package.nix:47".to_string() + ) + ) + .unwrap(), + Url::parse("https://example.com/some/path/pkgs/by-name/he/hello/package.nix#L47").unwrap() + ); + // Suffix default.nix if url is referencing folder assert_eq!( update_declaration( From 36a9525b224154409d05d3aaae0e5e542e142373 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Fri, 7 Nov 2025 01:06:27 +0100 Subject: [PATCH 011/136] Error on unknown fields --- ixx/src/action/index/mod.rs | 2 ++ ixx/src/option.rs | 3 +++ ixx/src/package.rs | 2 ++ 3 files changed, 7 insertions(+) diff --git a/ixx/src/action/index/mod.rs b/ixx/src/action/index/mod.rs index e81ee2d..de84100 100644 --- a/ixx/src/action/index/mod.rs +++ b/ixx/src/action/index/mod.rs @@ -14,12 +14,14 @@ mod options; mod packages; #[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] #[serde(rename_all = "camelCase")] pub(crate) struct Config { scopes: Vec, } #[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] #[serde(rename_all = "camelCase")] pub(crate) struct Scope { name: Option, diff --git a/ixx/src/option.rs b/ixx/src/option.rs index c160d12..5cbcccb 100644 --- a/ixx/src/option.rs +++ b/ixx/src/option.rs @@ -4,6 +4,7 @@ use url::Url; use crate::utils::highlight; #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] #[serde(rename_all = "camelCase")] pub struct Option { pub declarations: Vec, @@ -17,6 +18,7 @@ pub struct Option { } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] #[serde(rename_all = "camelCase", untagged)] pub enum Declaration { /// Example Value: `/nix/store/vgvk6q3zsjgb66f8s5cm8djz6nmcag1i-source/modules/initrd.nix` @@ -28,6 +30,7 @@ pub enum Declaration { } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] #[serde(rename_all = "camelCase")] #[serde(tag = "_type")] pub enum Content { diff --git a/ixx/src/package.rs b/ixx/src/package.rs index c72400d..f274feb 100644 --- a/ixx/src/package.rs +++ b/ixx/src/package.rs @@ -2,6 +2,7 @@ use serde::Deserialize; use url::Url; #[derive(Debug, Clone, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] #[serde(rename_all = "camelCase")] pub struct Package { pub attr_name: String, @@ -18,6 +19,7 @@ pub struct Package { } #[derive(Debug, Clone, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] #[serde(untagged)] pub enum OneOrMany { One(T), From ed77e31a7bb90789e280211f952353d89cfe56d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Fri, 7 Nov 2025 01:14:09 +0100 Subject: [PATCH 012/136] Add declaration --- ixx/src/action/index/packages.rs | 1 + ixx/src/package.rs | 1 + libixx/src/package.rs | 1 + 3 files changed, 3 insertions(+) diff --git a/ixx/src/action/index/packages.rs b/ixx/src/action/index/packages.rs index 0d4a2e9..78358f4 100644 --- a/ixx/src/action/index/packages.rs +++ b/ixx/src/action/index/packages.rs @@ -157,6 +157,7 @@ fn into_package(package: package::Package) -> anyhow::Result { attr_name: package.attr_name, eval_error: package.eval_error, broken: package.broken, + declaration: package.declaration, description: package.description, homepages: match package.homepage { None => vec![], diff --git a/ixx/src/package.rs b/ixx/src/package.rs index f274feb..4a7af6a 100644 --- a/ixx/src/package.rs +++ b/ixx/src/package.rs @@ -8,6 +8,7 @@ pub struct Package { pub attr_name: String, pub eval_error: Option, pub broken: Option, + pub declaration: Option, pub description: Option, pub homepage: Option>, pub outputs: Option>, diff --git a/libixx/src/package.rs b/libixx/src/package.rs index 09c5d89..5675157 100644 --- a/libixx/src/package.rs +++ b/libixx/src/package.rs @@ -6,6 +6,7 @@ pub struct Package { pub attr_name: String, pub eval_error: Option, pub broken: Option, + pub declaration: Option, pub description: Option, pub homepages: Vec, pub outputs: Vec, From b39b9cd50830087599babeac1cf38cdb77d72a00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Fri, 7 Nov 2025 01:25:13 +0100 Subject: [PATCH 013/136] Add licenses --- ixx/src/action/index/packages.rs | 1 + ixx/src/package.rs | 1 + libixx/src/package.rs | 1 + 3 files changed, 3 insertions(+) diff --git a/ixx/src/action/index/packages.rs b/ixx/src/action/index/packages.rs index 78358f4..9c5a7e8 100644 --- a/ixx/src/action/index/packages.rs +++ b/ixx/src/action/index/packages.rs @@ -164,6 +164,7 @@ fn into_package(package: package::Package) -> anyhow::Result { Some(OneOrMany::One(homepage)) => vec![homepage], Some(OneOrMany::Many(homepages)) => homepages, }, + licenses: package.licenses.unwrap_or_default(), outputs: package.outputs.unwrap_or_default(), insecure: package.insecure, name: package.name, diff --git a/ixx/src/package.rs b/ixx/src/package.rs index 4a7af6a..2e19ed0 100644 --- a/ixx/src/package.rs +++ b/ixx/src/package.rs @@ -11,6 +11,7 @@ pub struct Package { pub declaration: Option, pub description: Option, pub homepage: Option>, + pub licenses: Option>, pub outputs: Option>, pub insecure: Option, pub name: Option, diff --git a/libixx/src/package.rs b/libixx/src/package.rs index 5675157..e58f14f 100644 --- a/libixx/src/package.rs +++ b/libixx/src/package.rs @@ -9,6 +9,7 @@ pub struct Package { pub declaration: Option, pub description: Option, pub homepages: Vec, + pub licenses: Vec, pub outputs: Vec, pub insecure: Option, pub name: Option, From 08f66bccc8a1895d65902707847847669cfffded Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Sat, 8 Nov 2025 00:27:52 +0100 Subject: [PATCH 014/136] Add clippy --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index 622c899..8b45bc0 100644 --- a/flake.nix +++ b/flake.nix @@ -17,6 +17,7 @@ devShells.default = pkgs.mkShell { nativeBuildInputs = with pkgs; [ cargo + clippy rustc rustc.llvmPackages.lld wasm-pack From 31af1a234ede489b4a879f4bb119611c163698a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Sat, 8 Nov 2025 00:28:02 +0100 Subject: [PATCH 015/136] Add direnv --- .envrc | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .envrc diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..82b2b9e --- /dev/null +++ b/.envrc @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# the shebang is ignored, but nice for editors + +if type -P lorri &>/dev/null; then + eval "$(lorri direnv)" +else + echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]' + use nix +fi From 2d4ff0249e9f884b31fdbb1896d0e36c708c3799 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Sat, 8 Nov 2025 00:28:10 +0100 Subject: [PATCH 016/136] Add new mapping fields --- ixx/src/action/index/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ixx/src/action/index/mod.rs b/ixx/src/action/index/mod.rs index de84100..2c1b363 100644 --- a/ixx/src/action/index/mod.rs +++ b/ixx/src/action/index/mod.rs @@ -25,6 +25,8 @@ pub(crate) struct Config { #[serde(rename_all = "camelCase")] pub(crate) struct Scope { name: Option, + license_mapping: Option, + maintainer_mapping: Option, options_json: Option, packages_jsons: Option>, url_prefix: Url, From 583512acb2f4bf2b30dd74affbb495995ee4159d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Sat, 8 Nov 2025 00:41:31 +0100 Subject: [PATCH 017/136] Add maintainers --- ixx/src/action/index/packages.rs | 1 + ixx/src/package.rs | 1 + libixx/src/package.rs | 1 + 3 files changed, 3 insertions(+) diff --git a/ixx/src/action/index/packages.rs b/ixx/src/action/index/packages.rs index 9c5a7e8..6e3b2a5 100644 --- a/ixx/src/action/index/packages.rs +++ b/ixx/src/action/index/packages.rs @@ -164,6 +164,7 @@ fn into_package(package: package::Package) -> anyhow::Result { Some(OneOrMany::One(homepage)) => vec![homepage], Some(OneOrMany::Many(homepages)) => homepages, }, + maintainers: package.maintainers.unwrap_or_default(), licenses: package.licenses.unwrap_or_default(), outputs: package.outputs.unwrap_or_default(), insecure: package.insecure, diff --git a/ixx/src/package.rs b/ixx/src/package.rs index 2e19ed0..485ff84 100644 --- a/ixx/src/package.rs +++ b/ixx/src/package.rs @@ -11,6 +11,7 @@ pub struct Package { pub declaration: Option, pub description: Option, pub homepage: Option>, + pub maintainers: Option>, pub licenses: Option>, pub outputs: Option>, pub insecure: Option, diff --git a/libixx/src/package.rs b/libixx/src/package.rs index e58f14f..c39696a 100644 --- a/libixx/src/package.rs +++ b/libixx/src/package.rs @@ -10,6 +10,7 @@ pub struct Package { pub description: Option, pub homepages: Vec, pub licenses: Vec, + pub maintainers: Vec, pub outputs: Vec, pub insecure: Option, pub name: Option, From 8fb194d351f0d1408fb8e497ff111e3b278d0a42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Sat, 8 Nov 2025 00:58:36 +0100 Subject: [PATCH 018/136] Sort, add teams --- ixx/src/action/index/packages.rs | 9 +++++---- ixx/src/package.rs | 9 +++++---- libixx/src/package.rs | 7 ++++--- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/ixx/src/action/index/packages.rs b/ixx/src/action/index/packages.rs index 6e3b2a5..c46e359 100644 --- a/ixx/src/action/index/packages.rs +++ b/ixx/src/action/index/packages.rs @@ -155,21 +155,22 @@ pub(crate) async fn index_packages(module: &IndexModule, config: &Config) -> any fn into_package(package: package::Package) -> anyhow::Result { Ok(libixx::Package { attr_name: package.attr_name, - eval_error: package.eval_error, broken: package.broken, declaration: package.declaration, description: package.description, + eval_error: package.eval_error, homepages: match package.homepage { None => vec![], Some(OneOrMany::One(homepage)) => vec![homepage], Some(OneOrMany::Many(homepages)) => homepages, }, - maintainers: package.maintainers.unwrap_or_default(), - licenses: package.licenses.unwrap_or_default(), - outputs: package.outputs.unwrap_or_default(), insecure: package.insecure, + licenses: package.licenses.unwrap_or_default(), + maintainers: package.maintainers.unwrap_or_default(), name: package.name, + outputs: package.outputs.unwrap_or_default(), pname: package.pname, + teams: package.teams, unfree: package.unfree, version: package.version, }) diff --git a/ixx/src/package.rs b/ixx/src/package.rs index 485ff84..5e9a797 100644 --- a/ixx/src/package.rs +++ b/ixx/src/package.rs @@ -6,17 +6,18 @@ use url::Url; #[serde(rename_all = "camelCase")] pub struct Package { pub attr_name: String, - pub eval_error: Option, pub broken: Option, pub declaration: Option, pub description: Option, + pub eval_error: Option, pub homepage: Option>, - pub maintainers: Option>, - pub licenses: Option>, - pub outputs: Option>, pub insecure: Option, + pub licenses: Option>, + pub maintainers: Option>, pub name: Option, + pub outputs: Option>, pub pname: Option, + pub teams: Option, pub unfree: Option, pub version: Option, } diff --git a/libixx/src/package.rs b/libixx/src/package.rs index c39696a..a94271d 100644 --- a/libixx/src/package.rs +++ b/libixx/src/package.rs @@ -4,17 +4,18 @@ use url::Url; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct Package { pub attr_name: String, - pub eval_error: Option, pub broken: Option, pub declaration: Option, pub description: Option, + pub eval_error: Option, pub homepages: Vec, + pub insecure: Option, pub licenses: Vec, pub maintainers: Vec, - pub outputs: Vec, - pub insecure: Option, pub name: Option, + pub outputs: Vec, pub pname: Option, + pub teams: Option, pub unfree: Option, pub version: Option, } From fe7235c63f4ef87cbcd1dabfc4c65bd1302edd04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Sat, 8 Nov 2025 01:14:16 +0100 Subject: [PATCH 019/136] Make teams Vec --- ixx/src/action/index/packages.rs | 2 +- ixx/src/package.rs | 2 +- libixx/src/package.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ixx/src/action/index/packages.rs b/ixx/src/action/index/packages.rs index c46e359..0ef88e4 100644 --- a/ixx/src/action/index/packages.rs +++ b/ixx/src/action/index/packages.rs @@ -170,7 +170,7 @@ fn into_package(package: package::Package) -> anyhow::Result { name: package.name, outputs: package.outputs.unwrap_or_default(), pname: package.pname, - teams: package.teams, + teams: package.teams.unwrap_or_default(), unfree: package.unfree, version: package.version, }) diff --git a/ixx/src/package.rs b/ixx/src/package.rs index 5e9a797..d65434d 100644 --- a/ixx/src/package.rs +++ b/ixx/src/package.rs @@ -17,7 +17,7 @@ pub struct Package { pub name: Option, pub outputs: Option>, pub pname: Option, - pub teams: Option, + pub teams: Option>, pub unfree: Option, pub version: Option, } diff --git a/libixx/src/package.rs b/libixx/src/package.rs index a94271d..17560e9 100644 --- a/libixx/src/package.rs +++ b/libixx/src/package.rs @@ -15,7 +15,7 @@ pub struct Package { pub name: Option, pub outputs: Vec, pub pname: Option, - pub teams: Option, + pub teams: Vec, pub unfree: Option, pub version: Option, } From 44df433682e490e836c55dd7850779aa328b1248 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Sat, 8 Nov 2025 02:49:07 +0100 Subject: [PATCH 020/136] Drop unfree --- ixx/src/action/index/packages.rs | 1 - ixx/src/package.rs | 1 - libixx/src/package.rs | 1 - 3 files changed, 3 deletions(-) diff --git a/ixx/src/action/index/packages.rs b/ixx/src/action/index/packages.rs index 0ef88e4..726901a 100644 --- a/ixx/src/action/index/packages.rs +++ b/ixx/src/action/index/packages.rs @@ -171,7 +171,6 @@ fn into_package(package: package::Package) -> anyhow::Result { outputs: package.outputs.unwrap_or_default(), pname: package.pname, teams: package.teams.unwrap_or_default(), - unfree: package.unfree, version: package.version, }) } diff --git a/ixx/src/package.rs b/ixx/src/package.rs index d65434d..4447581 100644 --- a/ixx/src/package.rs +++ b/ixx/src/package.rs @@ -18,7 +18,6 @@ pub struct Package { pub outputs: Option>, pub pname: Option, pub teams: Option>, - pub unfree: Option, pub version: Option, } diff --git a/libixx/src/package.rs b/libixx/src/package.rs index 17560e9..88cf907 100644 --- a/libixx/src/package.rs +++ b/libixx/src/package.rs @@ -16,6 +16,5 @@ pub struct Package { pub outputs: Vec, pub pname: Option, pub teams: Vec, - pub unfree: Option, pub version: Option, } From 815754ece6c6cc386d78d51a6fdc1895c086609d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Sat, 8 Nov 2025 02:53:27 +0100 Subject: [PATCH 021/136] Replace insecure with known_vulnerabilities --- ixx/src/action/index/packages.rs | 2 +- ixx/src/package.rs | 2 +- libixx/src/package.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ixx/src/action/index/packages.rs b/ixx/src/action/index/packages.rs index 726901a..4efdd5a 100644 --- a/ixx/src/action/index/packages.rs +++ b/ixx/src/action/index/packages.rs @@ -164,7 +164,7 @@ fn into_package(package: package::Package) -> anyhow::Result { Some(OneOrMany::One(homepage)) => vec![homepage], Some(OneOrMany::Many(homepages)) => homepages, }, - insecure: package.insecure, + known_vulnerabilities: package.known_vulnerabilities.unwrap_or_default(), licenses: package.licenses.unwrap_or_default(), maintainers: package.maintainers.unwrap_or_default(), name: package.name, diff --git a/ixx/src/package.rs b/ixx/src/package.rs index 4447581..8d413ab 100644 --- a/ixx/src/package.rs +++ b/ixx/src/package.rs @@ -11,7 +11,7 @@ pub struct Package { pub description: Option, pub eval_error: Option, pub homepage: Option>, - pub insecure: Option, + pub known_vulnerabilities: Option>, pub licenses: Option>, pub maintainers: Option>, pub name: Option, diff --git a/libixx/src/package.rs b/libixx/src/package.rs index 88cf907..a1086c6 100644 --- a/libixx/src/package.rs +++ b/libixx/src/package.rs @@ -9,7 +9,7 @@ pub struct Package { pub description: Option, pub eval_error: Option, pub homepages: Vec, - pub insecure: Option, + pub known_vulnerabilities: Vec, pub licenses: Vec, pub maintainers: Vec, pub name: Option, From 367fb24b4ede4c3ba4776c44d6387aec89546e9d Mon Sep 17 00:00:00 2001 From: Marcel Date: Sat, 8 Nov 2025 04:05:10 +0100 Subject: [PATCH 022/136] packages: convert CVEs to links --- Cargo.lock | 9 +++++---- ixx/Cargo.toml | 1 + ixx/src/action/index/packages.rs | 21 +++++++++++++++++++-- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6c4f90c..044c09b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -365,6 +365,7 @@ dependencies = [ "clap", "libixx", "markdown", + "regex", "serde", "serde_json", "tokio", @@ -470,9 +471,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.11.1" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" dependencies = [ "aho-corasick", "memchr", @@ -482,9 +483,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.9" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", diff --git a/ixx/Cargo.toml b/ixx/Cargo.toml index 3fefc3e..75437d1 100644 --- a/ixx/Cargo.toml +++ b/ixx/Cargo.toml @@ -18,6 +18,7 @@ libixx = { path = "../libixx" } markdown = "1.0" serde_json = "1.0" anyhow = "1.0" +regex = "1.12" tree-sitter-highlight = "0.25" # when updating commit, also update HIGHLIGHT_NAMES in highlight.rs diff --git a/ixx/src/action/index/packages.rs b/ixx/src/action/index/packages.rs index 4efdd5a..bc41f34 100644 --- a/ixx/src/action/index/packages.rs +++ b/ixx/src/action/index/packages.rs @@ -1,7 +1,8 @@ -use std::io::Cursor; +use std::{io::Cursor, sync::LazyLock}; use anyhow::Context; use libixx::{Index, IndexBuilder}; +use regex::{Captures, Regex}; use tokio::{fs::File, io::AsyncWriteExt, task::JoinSet}; use crate::{ @@ -153,6 +154,8 @@ pub(crate) async fn index_packages(module: &IndexModule, config: &Config) -> any } fn into_package(package: package::Package) -> anyhow::Result { + static CVE_REGEX: LazyLock = LazyLock::new(|| Regex::new(r"CVE-(\d{4})-(\d+)").unwrap()); + Ok(libixx::Package { attr_name: package.attr_name, broken: package.broken, @@ -164,7 +167,21 @@ fn into_package(package: package::Package) -> anyhow::Result { Some(OneOrMany::One(homepage)) => vec![homepage], Some(OneOrMany::Many(homepages)) => homepages, }, - known_vulnerabilities: package.known_vulnerabilities.unwrap_or_default(), + known_vulnerabilities: package + .known_vulnerabilities + .unwrap_or_default() + .into_iter() + .map(|vulnerability| { + CVE_REGEX + .replace_all(&vulnerability, |caps: &Captures| { + format!( + "CVE-{0}-{1}", + &caps[1], &caps[2] + ) + }) + .to_string() + }) + .collect(), licenses: package.licenses.unwrap_or_default(), maintainers: package.maintainers.unwrap_or_default(), name: package.name, From dd61cda64e6e6ac48b772287545619dfa406c86d Mon Sep 17 00:00:00 2001 From: Marcel Date: Sat, 8 Nov 2025 04:42:16 +0100 Subject: [PATCH 023/136] packages: also update declerations --- ixx/src/action/index/mod.rs | 138 +++++++++++++++++++++++++++++- ixx/src/action/index/options.rs | 140 +------------------------------ ixx/src/action/index/packages.rs | 17 ++-- ixx/src/main.rs | 14 ++++ ixx/src/option.rs | 17 +--- ixx/src/package.rs | 4 +- libixx/src/package.rs | 2 +- 7 files changed, 172 insertions(+), 160 deletions(-) diff --git a/ixx/src/action/index/mod.rs b/ixx/src/action/index/mod.rs index 2c1b363..eacf934 100644 --- a/ixx/src/action/index/mod.rs +++ b/ixx/src/action/index/mod.rs @@ -1,11 +1,12 @@ use std::path::PathBuf; -use anyhow::Context; +use anyhow::{Context, anyhow}; use serde::Deserialize; use tokio::join; use url::Url; use crate::{ + Declaration, action::index::{options::index_options, packages::index_packages}, args::IndexModule, }; @@ -68,3 +69,138 @@ pub(crate) async fn index(module: IndexModule) -> anyhow::Result<()> { Ok(()) } + +fn update_declaration(url_prefix: &Url, declaration: Declaration) -> anyhow::Result { + let mut url = match declaration { + Declaration::StorePath(path) => { + let mut url_path; + if path.starts_with("/") { + let idx = path + .match_indices('/') + .nth(3) + .ok_or_else(|| anyhow!("Invalid store path: {}", path))? + .0 + // +1 to also remove the / itself, when we join it with a url, the path in the url would + // get removed if we won't remove it. + + 1; + url_path = path.split_at(idx).1.to_owned(); + } else { + url_path = path + } + + if let Some((path, line)) = url_path.split_once(':') { + url_path = format!("{path}#L{line}"); + } + + url_prefix.join(&url_path)? + } + Declaration::Url { name: _, url } => url, + }; + + if !url.path().ends_with(".nix") { + if url.path().ends_with("/") { + url = url.join("default.nix")?; + } else { + url = url.join(&format!( + "{}/default.nix", + url + .path_segments() + .map(|mut segments| segments.next_back().unwrap_or("")) + .unwrap_or(""), + ))?; + } + } + + Ok(url) +} + +#[cfg(test)] +mod test { + use url::Url; + + use crate::{Declaration, action::index::update_declaration}; + + #[test] + fn test_update_declaration() { + assert_eq!( + update_declaration( + &Url::parse("https://example.com/some/path").unwrap(), + Declaration::StorePath( + "/nix/store/vgvk6q3zsjgb66f8s5cm8djz6nmcag1i-source/modules/initrd.nix".to_string() + ) + ) + .unwrap(), + Url::parse("https://example.com/some/modules/initrd.nix").unwrap() + ); + + assert_eq!( + update_declaration( + &Url::parse("https://example.com/some/path/").unwrap(), + Declaration::StorePath( + "/nix/store/vgvk6q3zsjgb66f8s5cm8djz6nmcag1i-source/modules/initrd.nix".to_string() + ) + ) + .unwrap(), + Url::parse("https://example.com/some/path/modules/initrd.nix").unwrap() + ); + + assert_eq!( + update_declaration( + &Url::parse("https://example.com/some/path/").unwrap(), + Declaration::StorePath( + "/nix/store/vgvk6q3zsjgb66f8s5cm8djz6nmcag1i-source-idk/modules/initrd.nix".to_string() + ) + ) + .unwrap(), + Url::parse("https://example.com/some/path/modules/initrd.nix").unwrap() + ); + + // package position + assert_eq!( + update_declaration( + &Url::parse("https://example.com/some/path/").unwrap(), + Declaration::StorePath( + "/nix/store/pb93n2bk2zpyn1sqpkm3gyhra26zy4ps-source/pkgs/by-name/he/hello/package.nix:47" + .to_string() + ) + ) + .unwrap(), + Url::parse("https://example.com/some/path/pkgs/by-name/he/hello/package.nix#L47").unwrap() + ); + + // Suffix default.nix if url is referencing folder + assert_eq!( + update_declaration( + &Url::parse("https://example.com/some/path").unwrap(), + Declaration::Url { + name: "idk".to_string(), + url: Url::parse("https://example.com/some/path").unwrap(), + } + ) + .unwrap(), + Url::parse("https://example.com/some/path/default.nix").unwrap() + ); + + assert_eq!( + update_declaration( + &Url::parse("https://example.com/some/path").unwrap(), + Declaration::Url { + name: "idk".to_string(), + url: Url::parse("https://example.com/some/path/").unwrap(), + } + ) + .unwrap(), + Url::parse("https://example.com/some/path/default.nix").unwrap() + ); + + // nixpkgs edge case + assert_eq!( + update_declaration( + &Url::parse("https://example.com/some/path/").unwrap(), + Declaration::StorePath("nixos/hello/world.nix".to_string()), + ) + .unwrap(), + Url::parse("https://example.com/some/path/nixos/hello/world.nix").unwrap() + ); + } +} diff --git a/ixx/src/action/index/options.rs b/ixx/src/action/index/options.rs index 542aac0..3e272dc 100644 --- a/ixx/src/action/index/options.rs +++ b/ixx/src/action/index/options.rs @@ -1,14 +1,14 @@ use std::{collections::HashMap, io::Cursor}; -use anyhow::{Context, anyhow}; +use anyhow::Context; use libixx::{Index, IndexBuilder}; use tokio::{fs::File, io::AsyncWriteExt, task::JoinSet}; use url::Url; use crate::{ - action::index::{Config, OptionEntry}, + action::index::{Config, OptionEntry, update_declaration}, args::IndexModule, - option::{self, Declaration}, + option::{self}, }; pub(crate) async fn index_options(module: &IndexModule, config: &Config) -> anyhow::Result<()> { @@ -175,137 +175,3 @@ fn into_option( name: name.to_string(), }) } - -fn update_declaration(url_prefix: &Url, declaration: Declaration) -> anyhow::Result { - let mut url = match declaration { - Declaration::StorePath(path) => { - let mut url_path; - if path.starts_with("/") { - let idx = path - .match_indices('/') - .nth(3) - .ok_or_else(|| anyhow!("Invalid store path: {}", path))? - .0 - // +1 to also remove the / itself, when we join it with a url, the path in the url would - // get removed if we won't remove it. - + 1; - url_path = path.split_at(idx).1.to_owned(); - } else { - url_path = path - } - - if let Some((path, line)) = url_path.split_once(':') { - url_path = format!("{path}#L{line}"); - } - - url_prefix.join(&url_path)? - } - Declaration::Url { name: _, url } => url, - }; - - if !url.path().ends_with(".nix") { - if url.path().ends_with("/") { - url = url.join("default.nix")?; - } else { - url = url.join(&format!( - "{}/default.nix", - url - .path_segments() - .map(|mut segments| segments.next_back().unwrap_or("")) - .unwrap_or(""), - ))?; - } - } - - Ok(url) -} - -#[cfg(test)] -mod test { - use url::Url; - - use crate::{action::index::options::update_declaration, option::Declaration}; - - #[test] - fn test_update_declaration() { - assert_eq!( - update_declaration( - &Url::parse("https://example.com/some/path").unwrap(), - Declaration::StorePath( - "/nix/store/vgvk6q3zsjgb66f8s5cm8djz6nmcag1i-source/modules/initrd.nix".to_string() - ) - ) - .unwrap(), - Url::parse("https://example.com/some/modules/initrd.nix").unwrap() - ); - - assert_eq!( - update_declaration( - &Url::parse("https://example.com/some/path/").unwrap(), - Declaration::StorePath( - "/nix/store/vgvk6q3zsjgb66f8s5cm8djz6nmcag1i-source/modules/initrd.nix".to_string() - ) - ) - .unwrap(), - Url::parse("https://example.com/some/path/modules/initrd.nix").unwrap() - ); - - assert_eq!( - update_declaration( - &Url::parse("https://example.com/some/path/").unwrap(), - Declaration::StorePath( - "/nix/store/vgvk6q3zsjgb66f8s5cm8djz6nmcag1i-source-idk/modules/initrd.nix".to_string() - ) - ) - .unwrap(), - Url::parse("https://example.com/some/path/modules/initrd.nix").unwrap() - ); - - // package position - assert_eq!( - update_declaration( - &Url::parse("https://example.com/some/path/").unwrap(), - Declaration::StorePath( - "/nix/store/pb93n2bk2zpyn1sqpkm3gyhra26zy4ps-source/pkgs/by-name/he/hello/package.nix:47".to_string() - ) - ) - .unwrap(), - Url::parse("https://example.com/some/path/pkgs/by-name/he/hello/package.nix#L47").unwrap() - ); - - // Suffix default.nix if url is referencing folder - assert_eq!( - update_declaration( - &Url::parse("https://example.com/some/path").unwrap(), - Declaration::Url { - name: "idk".to_string(), - url: Url::parse("https://example.com/some/path").unwrap(), - } - ) - .unwrap(), - Url::parse("https://example.com/some/path/default.nix").unwrap() - ); - - assert_eq!( - update_declaration( - &Url::parse("https://example.com/some/path").unwrap(), - Declaration::Url { - name: "idk".to_string(), - url: Url::parse("https://example.com/some/path/").unwrap(), - } - ) - .unwrap(), - Url::parse("https://example.com/some/path/default.nix").unwrap() - ); - - // nixpkgs edge case - assert_eq!( - update_declaration( - &Url::parse("https://example.com/some/path/").unwrap(), - Declaration::StorePath("nixos/hello/world.nix".to_string()), - ) - .unwrap(), - Url::parse("https://example.com/some/path/nixos/hello/world.nix").unwrap() - ); - } -} diff --git a/ixx/src/action/index/packages.rs b/ixx/src/action/index/packages.rs index bc41f34..a955cd1 100644 --- a/ixx/src/action/index/packages.rs +++ b/ixx/src/action/index/packages.rs @@ -1,12 +1,16 @@ -use std::{io::Cursor, sync::LazyLock}; +use std::{ + io::Cursor, + sync::{Arc, LazyLock}, +}; use anyhow::Context; use libixx::{Index, IndexBuilder}; use regex::{Captures, Regex}; use tokio::{fs::File, io::AsyncWriteExt, task::JoinSet}; +use url::Url; use crate::{ - action::index::{Config, PackageEntry}, + action::index::{Config, PackageEntry, update_declaration}, args::IndexModule, package::{self, OneOrMany}, }; @@ -34,8 +38,11 @@ pub(crate) async fn index_packages(module: &IndexModule, config: &Config) -> any let mut join_set = JoinSet::new(); + let url_prefix = Arc::new(scope.url_prefix.clone()); + for packages_json in packages_jsons { let packages_json = packages_json.clone(); + let url_prefix = url_prefix.clone(); join_set.spawn(async move { println!("Parsing {}", packages_json.to_string_lossy()); let packages: Vec = { @@ -56,7 +63,7 @@ pub(crate) async fn index_packages(module: &IndexModule, config: &Config) -> any Ok::<_, anyhow::Error>(PackageEntry { name: package.attr_name.clone(), scope: scope_idx, - option: into_package(package)?, + option: into_package(&url_prefix, package)?, }) }) .collect::, _>>()?; @@ -153,13 +160,13 @@ pub(crate) async fn index_packages(module: &IndexModule, config: &Config) -> any Ok(()) } -fn into_package(package: package::Package) -> anyhow::Result { +fn into_package(url_prefix: &Url, package: package::Package) -> anyhow::Result { static CVE_REGEX: LazyLock = LazyLock::new(|| Regex::new(r"CVE-(\d{4})-(\d+)").unwrap()); Ok(libixx::Package { attr_name: package.attr_name, broken: package.broken, - declaration: package.declaration, + declaration: package.declaration.map(|declaration | update_declaration(url_prefix, declaration)).transpose()?, description: package.description, eval_error: package.eval_error, homepages: match package.homepage { diff --git a/ixx/src/main.rs b/ixx/src/main.rs index 547fb31..f1481fb 100644 --- a/ixx/src/main.rs +++ b/ixx/src/main.rs @@ -1,5 +1,7 @@ use args::{Action, Args}; use clap::Parser; +use serde::Deserialize; +use url::Url; mod action; mod args; @@ -19,3 +21,15 @@ async fn main() -> anyhow::Result<()> { Ok(()) } + +#[derive(Debug, Clone, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] +#[serde(rename_all = "camelCase", untagged)] +pub enum Declaration { + /// Example Value: `/nix/store/vgvk6q3zsjgb66f8s5cm8djz6nmcag1i-source/modules/initrd.nix` + StorePath(String), + Url { + name: String, + url: Url, + }, +} diff --git a/ixx/src/option.rs b/ixx/src/option.rs index 5cbcccb..69c4287 100644 --- a/ixx/src/option.rs +++ b/ixx/src/option.rs @@ -1,9 +1,8 @@ use serde::{Deserialize, Serialize}; -use url::Url; -use crate::utils::highlight; +use crate::{Declaration, utils::highlight}; -#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Default, Debug, Clone, PartialEq, Deserialize)] #[serde(deny_unknown_fields)] #[serde(rename_all = "camelCase")] pub struct Option { @@ -17,18 +16,6 @@ pub struct Option { pub related_packages: std::option::Option, } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -#[serde(rename_all = "camelCase", untagged)] -pub enum Declaration { - /// Example Value: `/nix/store/vgvk6q3zsjgb66f8s5cm8djz6nmcag1i-source/modules/initrd.nix` - StorePath(String), - Url { - name: String, - url: Url, - }, -} - #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] #[serde(rename_all = "camelCase")] diff --git a/ixx/src/package.rs b/ixx/src/package.rs index 8d413ab..44ef670 100644 --- a/ixx/src/package.rs +++ b/ixx/src/package.rs @@ -1,13 +1,15 @@ use serde::Deserialize; use url::Url; +use crate::Declaration; + #[derive(Debug, Clone, PartialEq, Deserialize)] #[serde(deny_unknown_fields)] #[serde(rename_all = "camelCase")] pub struct Package { pub attr_name: String, pub broken: Option, - pub declaration: Option, + pub declaration: Option, pub description: Option, pub eval_error: Option, pub homepage: Option>, diff --git a/libixx/src/package.rs b/libixx/src/package.rs index a1086c6..5c9fb36 100644 --- a/libixx/src/package.rs +++ b/libixx/src/package.rs @@ -5,7 +5,7 @@ use url::Url; pub struct Package { pub attr_name: String, pub broken: Option, - pub declaration: Option, + pub declaration: Option, pub description: Option, pub eval_error: Option, pub homepages: Vec, From 43898294162913be1bf1c1a5d52235f9c40958f5 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sat, 8 Nov 2025 05:19:41 +0100 Subject: [PATCH 024/136] meta: add licesences and maintainer meta --- ixx/src/action/index/mod.rs | 72 ++++++++++++++++++++++++++++++++----- ixx/src/args.rs | 3 ++ 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/ixx/src/action/index/mod.rs b/ixx/src/action/index/mod.rs index eacf934..af47227 100644 --- a/ixx/src/action/index/mod.rs +++ b/ixx/src/action/index/mod.rs @@ -1,8 +1,8 @@ -use std::path::PathBuf; +use std::{collections::HashMap, path::PathBuf}; -use anyhow::{Context, anyhow}; -use serde::Deserialize; -use tokio::join; +use anyhow::Context; +use serde::{Deserialize, Serialize}; +use tokio::{fs::File, io::AsyncWriteExt, join}; use url::Url; use crate::{ @@ -26,8 +26,8 @@ pub(crate) struct Config { #[serde(rename_all = "camelCase")] pub(crate) struct Scope { name: Option, - license_mapping: Option, - maintainer_mapping: Option, + license_mapping: HashMap, + maintainer_mapping: HashMap, options_json: Option, packages_jsons: Option>, url_prefix: Url, @@ -46,6 +46,36 @@ struct PackageEntry { option: libixx::Package, } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct License { + free: bool, + full_name: String, + redistributable: bool, + spdx_id: String, + url: Url, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Maintainer { + email: String, + github: String, + github_id: u32, + name: String, +} + +#[derive(Serialize)] +struct Meta { + scopes: HashMap, +} + +#[derive(Serialize)] +struct ScopeMeta { + licenses: HashMap, + maintainers: HashMap, +} + pub(crate) async fn index(module: IndexModule) -> anyhow::Result<()> { let config: Config = { let raw_config = tokio::fs::read_to_string(&module.config) @@ -59,13 +89,39 @@ pub(crate) async fn index(module: IndexModule) -> anyhow::Result<()> { serde_json::from_str(&raw_config)? }; - let (options_result, packages_result) = join!( + let (options_result, packages_result, meta_result) = join!( index_options(&module, &config), index_packages(&module, &config), + async { + let meta = Meta { + scopes: config + .scopes + .iter() + .enumerate() + .map(|(idx, scope)| { + ( + idx as u8, + ScopeMeta { + licenses: scope.license_mapping.clone(), + maintainers: scope.maintainer_mapping.clone(), + }, + ) + }) + .collect(), + }; + + let raw_meta = serde_json::to_string(&meta)?; + let mut meta_file = File::create(&module.meta_output).await?; + + meta_file.write_all(raw_meta.as_bytes()).await?; + + Ok::<_, anyhow::Error>(()) + } ); options_result?; packages_result?; + meta_result?; Ok(()) } @@ -78,7 +134,7 @@ fn update_declaration(url_prefix: &Url, declaration: Declaration) -> anyhow::Res let idx = path .match_indices('/') .nth(3) - .ok_or_else(|| anyhow!("Invalid store path: {}", path))? + .ok_or_else(|| anyhow::anyhow!("Invalid store path: {}", path))? .0 // +1 to also remove the / itself, when we join it with a url, the path in the url would // get removed if we won't remove it. diff --git a/ixx/src/args.rs b/ixx/src/args.rs index 766dc3e..d2a594d 100644 --- a/ixx/src/args.rs +++ b/ixx/src/args.rs @@ -36,6 +36,9 @@ pub(super) struct IndexModule { #[clap(long, default_value = "packages/meta")] pub(crate) packages_meta_output: PathBuf, + #[clap(long, default_value = "meta.json")] + pub(crate) meta_output: PathBuf, + #[clap(long, default_value = "100")] pub(super) chunk_size: u32, } From b91353d21ff80a899ac71fbe929a42d4c81a8d63 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sat, 8 Nov 2025 05:55:40 +0100 Subject: [PATCH 025/136] ixx: use camelCase --- ixx/src/action/index/mod.rs | 12 ++++++------ ixx/src/option.rs | 6 ++---- ixx/src/package.rs | 5 ++--- libixx/src/option.rs | 1 + libixx/src/package.rs | 1 + 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/ixx/src/action/index/mod.rs b/ixx/src/action/index/mod.rs index af47227..b0858d9 100644 --- a/ixx/src/action/index/mod.rs +++ b/ixx/src/action/index/mod.rs @@ -15,15 +15,13 @@ mod options; mod packages; #[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields, rename_all = "camelCase")] pub(crate) struct Config { scopes: Vec, } #[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields, rename_all = "camelCase")] pub(crate) struct Scope { name: Option, license_mapping: HashMap, @@ -47,7 +45,7 @@ struct PackageEntry { } #[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] struct License { free: bool, full_name: String, @@ -57,7 +55,7 @@ struct License { } #[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] struct Maintainer { email: String, github: String, @@ -66,11 +64,13 @@ struct Maintainer { } #[derive(Serialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] struct Meta { scopes: HashMap, } #[derive(Serialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] struct ScopeMeta { licenses: HashMap, maintainers: HashMap, diff --git a/ixx/src/option.rs b/ixx/src/option.rs index 69c4287..173fb34 100644 --- a/ixx/src/option.rs +++ b/ixx/src/option.rs @@ -3,8 +3,7 @@ use serde::{Deserialize, Serialize}; use crate::{Declaration, utils::highlight}; #[derive(Default, Debug, Clone, PartialEq, Deserialize)] -#[serde(deny_unknown_fields)] -#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields, rename_all = "camelCase")] pub struct Option { pub declarations: Vec, pub description: String, @@ -17,8 +16,7 @@ pub struct Option { } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields, rename_all = "camelCase")] #[serde(tag = "_type")] pub enum Content { LiteralExpression { diff --git a/ixx/src/package.rs b/ixx/src/package.rs index 44ef670..3c86aeb 100644 --- a/ixx/src/package.rs +++ b/ixx/src/package.rs @@ -4,8 +4,7 @@ use url::Url; use crate::Declaration; #[derive(Debug, Clone, PartialEq, Deserialize)] -#[serde(deny_unknown_fields)] -#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields, rename_all = "camelCase")] pub struct Package { pub attr_name: String, pub broken: Option, @@ -24,7 +23,7 @@ pub struct Package { } #[derive(Debug, Clone, PartialEq, Deserialize)] -#[serde(deny_unknown_fields)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] #[serde(untagged)] pub enum OneOrMany { One(T), diff --git a/libixx/src/option.rs b/libixx/src/option.rs index 19e1a02..598ac47 100644 --- a/libixx/src/option.rs +++ b/libixx/src/option.rs @@ -2,6 +2,7 @@ use serde::{Deserialize, Serialize}; use url::Url; #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] pub struct Option { pub declarations: Vec, pub default: std::option::Option, diff --git a/libixx/src/package.rs b/libixx/src/package.rs index 5c9fb36..b0d2c70 100644 --- a/libixx/src/package.rs +++ b/libixx/src/package.rs @@ -2,6 +2,7 @@ use serde::{Deserialize, Serialize}; use url::Url; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] pub struct Package { pub attr_name: String, pub broken: Option, From fca8b55714c922e4c8062c1daeadecf797dba51b Mon Sep 17 00:00:00 2001 From: Marcel Date: Sat, 8 Nov 2025 06:21:02 +0100 Subject: [PATCH 026/136] fix maintainers --- ixx/src/action/index/mod.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ixx/src/action/index/mod.rs b/ixx/src/action/index/mod.rs index b0858d9..8584edc 100644 --- a/ixx/src/action/index/mod.rs +++ b/ixx/src/action/index/mod.rs @@ -50,16 +50,17 @@ struct License { free: bool, full_name: String, redistributable: bool, - spdx_id: String, - url: Url, + spdx_id: Option, + url: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields, rename_all = "camelCase")] struct Maintainer { - email: String, + email: Option, + matrix: Option, github: String, - github_id: u32, + github_id: Option, name: String, } From 029a0e23ab6a5e86c24d66331cf8b1d0c167709e Mon Sep 17 00:00:00 2001 From: Marcel Date: Sat, 8 Nov 2025 20:12:46 +0100 Subject: [PATCH 027/136] ixx: remove github id --- ixx/src/action/index/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/ixx/src/action/index/mod.rs b/ixx/src/action/index/mod.rs index 8584edc..5e1b229 100644 --- a/ixx/src/action/index/mod.rs +++ b/ixx/src/action/index/mod.rs @@ -60,7 +60,6 @@ struct Maintainer { email: Option, matrix: Option, github: String, - github_id: Option, name: String, } From 2d5e76faf082878ae4536996ad0b3954276188ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Sat, 8 Nov 2025 20:42:37 +0100 Subject: [PATCH 028/136] Change id to u32 --- ixx/src/action/index/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ixx/src/action/index/mod.rs b/ixx/src/action/index/mod.rs index 5e1b229..3c6f62b 100644 --- a/ixx/src/action/index/mod.rs +++ b/ixx/src/action/index/mod.rs @@ -25,7 +25,7 @@ pub(crate) struct Config { pub(crate) struct Scope { name: Option, license_mapping: HashMap, - maintainer_mapping: HashMap, + maintainer_mapping: HashMap, options_json: Option, packages_jsons: Option>, url_prefix: Url, @@ -73,7 +73,7 @@ struct Meta { #[serde(deny_unknown_fields, rename_all = "camelCase")] struct ScopeMeta { licenses: HashMap, - maintainers: HashMap, + maintainers: HashMap, } pub(crate) async fn index(module: IndexModule) -> anyhow::Result<()> { From f4926bfb56d21d110d66cbaa549af230d4707148 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sat, 8 Nov 2025 21:21:17 +0100 Subject: [PATCH 029/136] ixx: add context to json parse errors --- ixx/src/action/index/mod.rs | 7 ++++++- ixx/src/action/index/options.rs | 7 ++++++- ixx/src/action/index/packages.rs | 9 +++++++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/ixx/src/action/index/mod.rs b/ixx/src/action/index/mod.rs index 3c6f62b..9e236ae 100644 --- a/ixx/src/action/index/mod.rs +++ b/ixx/src/action/index/mod.rs @@ -86,7 +86,12 @@ pub(crate) async fn index(module: IndexModule) -> anyhow::Result<()> { module.config.to_string_lossy() ) })?; - serde_json::from_str(&raw_config)? + serde_json::from_str(&raw_config).with_context(|| { + format!( + "Failed to parse config file: {}", + module.config.to_string_lossy() + ) + })? }; let (options_result, packages_result, meta_result) = join!( diff --git a/ixx/src/action/index/options.rs b/ixx/src/action/index/options.rs index 3e272dc..3f40d6b 100644 --- a/ixx/src/action/index/options.rs +++ b/ixx/src/action/index/options.rs @@ -34,7 +34,12 @@ pub(crate) async fn index_options(module: &IndexModule, config: &Config) -> anyh options_json.to_string_lossy() ) })?; - serde_json::from_str(&raw_options)? + serde_json::from_str(&raw_options).with_context(|| { + format!( + "Failed to parse options json: {}", + options_json.to_string_lossy() + ) + })? }; let scope_idx = index_builder.push_scope( diff --git a/ixx/src/action/index/packages.rs b/ixx/src/action/index/packages.rs index a955cd1..341cca3 100644 --- a/ixx/src/action/index/packages.rs +++ b/ixx/src/action/index/packages.rs @@ -50,11 +50,16 @@ pub(crate) async fn index_packages(module: &IndexModule, config: &Config) -> any .await .with_context(|| { format!( - "Failed to read options json: {}", + "Failed to read packages json: {}", packages_json.to_string_lossy() ) })?; - serde_json::from_str(&raw_packages)? + serde_json::from_str(&raw_packages).with_context(|| { + format!( + "Failed to parse packages json: {}", + packages_json.to_string_lossy() + ) + })? }; let packages = packages From 3b3edd00fcb40ccdc6c42387885174d6b65642d5 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sat, 8 Nov 2025 21:39:21 +0100 Subject: [PATCH 030/136] package: define maintainer as u32 github id --- ixx/src/package.rs | 2 +- libixx/src/package.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ixx/src/package.rs b/ixx/src/package.rs index 3c86aeb..b56e14b 100644 --- a/ixx/src/package.rs +++ b/ixx/src/package.rs @@ -14,7 +14,7 @@ pub struct Package { pub homepage: Option>, pub known_vulnerabilities: Option>, pub licenses: Option>, - pub maintainers: Option>, + pub maintainers: Option>, pub name: Option, pub outputs: Option>, pub pname: Option, diff --git a/libixx/src/package.rs b/libixx/src/package.rs index b0d2c70..905beea 100644 --- a/libixx/src/package.rs +++ b/libixx/src/package.rs @@ -12,7 +12,7 @@ pub struct Package { pub homepages: Vec, pub known_vulnerabilities: Vec, pub licenses: Vec, - pub maintainers: Vec, + pub maintainers: Vec, pub name: Option, pub outputs: Vec, pub pname: Option, From 93679d1c0c87175f484c1c742f43c44e94c643f7 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sat, 8 Nov 2025 23:11:17 +0100 Subject: [PATCH 031/136] ixx: handle path in literal expression --- ixx/src/option.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ixx/src/option.rs b/ixx/src/option.rs index 173fb34..d88e24a 100644 --- a/ixx/src/option.rs +++ b/ixx/src/option.rs @@ -21,17 +21,17 @@ pub struct Option { pub enum Content { LiteralExpression { text: String, + // nixvim uses this in programs.nixvim.dependencies.coreutils.package + path: std::option::Option>, }, #[serde(rename = "literalMD")] - Markdown { - text: String, - }, + Markdown { text: String }, } impl Content { pub(crate) fn render(self) -> String { match self { - Self::LiteralExpression { text } => highlight(text.trim()), + Self::LiteralExpression { text, .. } => highlight(text.trim()), Self::Markdown { text } => markdown::to_html(text.trim()), } } From b08bc0b533e1bd92ad17d1ae5867a5fd64619a26 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 9 Nov 2025 23:34:45 +0100 Subject: [PATCH 032/136] index: fix confition protecting against overflow --- libixx/src/index.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libixx/src/index.rs b/libixx/src/index.rs index d93a13f..77b968b 100644 --- a/libixx/src/index.rs +++ b/libixx/src/index.rs @@ -92,7 +92,7 @@ impl IndexBuilder { }); } - if self.index.entries.len() == u16::MAX.into() { + if self.index.entries.len() >= u16::MAX.into() { panic!( "You can not have more than 65535 entries. Please contact the developers for further assistance." ); From 1bd5e4ce4cec8eb8669b0f97c61f098268811981 Mon Sep 17 00:00:00 2001 From: Marcel Date: Mon, 10 Nov 2025 00:18:55 +0100 Subject: [PATCH 033/136] index: allow entry_idx up to u64 --- libixx/src/index.rs | 107 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 89 insertions(+), 18 deletions(-) diff --git a/libixx/src/index.rs b/libixx/src/index.rs index 77b968b..6ad44d5 100644 --- a/libixx/src/index.rs +++ b/libixx/src/index.rs @@ -9,11 +9,11 @@ use crate::IxxError; pub struct IndexBuilder { index: Index, - label_cache: HashMap, (u16, u8)>, + label_cache: HashMap, (usize, u8)>, } #[binrw] -#[brw(magic = b"ixx01")] +#[brw(magic = b"ixx02")] #[derive(Debug, Clone, PartialEq)] pub struct Index { meta: Meta, @@ -44,22 +44,98 @@ pub struct Entry { labels: Vec