Feat/add option to exclude fields

.github/workflows/build-test-and-lint.yml

@@ -73,6 +73,7 @@ jobs:
           rustup update ${{ matrix.channel }}
           rustup default ${{ matrix.channel }}
           rustup component add llvm-tools
+          rustup component add rustfmt
 
       - name: Check Rust ${{ matrix.channel }} toolchain versions
         shell: bash

.gitignore

@@ -181,3 +181,4 @@ Cargo.lock
 *.zip
 
 #endregion  ----------------------------------------------------------------------------------------
+.env

README.md

@@ -111,6 +111,9 @@ Options:
   -s, --select <field1,field2,...>
           Names of core fields retrieved in each document [default: all but _*]
 
+  -e, --exclude <field1,field2,...>
+          Names of core fields excluded in each document
+
   -i, --iterate-by <mode>
           Slice the queries by using the variables {begin} and {end} for iterating in `--query` Used in bigger solr cores with huge number of docs because querying the end of docs is expensive and fails frequently [default: day]
 

src/args.rs

@@ -83,9 +83,13 @@ pub(crate) struct Backup {
     pub limit: Option<u64>,
 
     /// Names of core fields retrieved in each document [default: all but _*]
-    #[arg(short, long, display_order = 45, value_name = "field1,field2,...")]
+    #[arg(short, long, display_order = 45, value_name = "field1,field2,...", value_parser = parse_trim, value_delimiter = ',')]
     pub select: Vec<String>,
 
+    /// Names of core fields excluded in each document [default: none]
+    #[arg(short, long, display_order = 46, value_name = "field1,field2,...", value_parser = parse_trim, value_delimiter = ',')]
+    pub exclude: Vec<String>,
+
     /// Slice the queries by using the variables {begin} and {end} for iterating in `--query`
     /// Used in bigger solr cores with huge number of docs because querying the end of docs is expensive and fails frequently
     #[arg(short, long, display_order = 50, default_value_t = IterateMode::Day, value_name = "mode", value_enum)]
@@ -346,6 +350,10 @@ pub(crate) const SOLR_COPY_URL: &str = "SOLR_COPY_URL";
 
 // #region Param parsing
 
+fn parse_trim(src: &str) -> Result<String, String> {
+    Ok(src.trim().to_string())
+}
+
 fn parse_quantity(src: &str) -> Result<u64, String> {
     lazy_static! {
         static ref REGKB: Regex =

src/main.rs

@@ -22,6 +22,7 @@
 // #![allow(unused_variables)]
 // #![allow(unused_imports)]
 // #![allow(dead_code)]
+// #![allow(unreachable_code)]
 
 // endregion
 

src/steps.rs

@@ -5,6 +5,9 @@ use super::{
     helpers::{IntegerHelpers, StringHelpers, replace_solr_date, solr_query},
 };
 use chrono::{DateTime, Duration, NaiveDate, NaiveDateTime, Utc};
+use log::debug;
+use std::collections::HashSet;
+use std::iter::FromIterator;
 
 // region Struct
 
@@ -292,19 +295,25 @@ impl Backup {
     }
 
     pub(crate) fn get_steps(&self, schema: &SolrCore) -> Requests {
-        let core_fields: &[String] = &schema.fields;
-        let fl = self.get_query_fields(core_fields);
+        let include_hash: HashSet<String> = HashSet::from_iter(schema.fields.clone());
+        let exclude_hash: HashSet<String> = HashSet::from_iter(self.exclude.clone());
+        let diff: Vec<String> = include_hash.difference(&exclude_hash).map(String::from).collect();
+
+        debug!("Include fields {:?}", include_hash);
+        debug!("Exclude fields {:?}", exclude_hash);
+        debug!("Actual fields  {:?}", diff);
+
+        let fl = self.get_query_fields(diff);
         let query = self.get_query_url(&fl, true);
         let end_limit = self.get_docs_to_retrieve(schema);
         Requests { curr: self.skip, limit: end_limit, num_docs: self.num_docs, url: query }
     }
 
-    pub(crate) fn get_query_fields(&self, core_fields: &[String]) -> String {
-        let fields = if self.select.is_empty() { core_fields } else { &self.select };
-        if fields.is_empty() {
+    pub(crate) fn get_query_fields(&self, core_fields: Vec<String>) -> String {
+        if core_fields.is_empty() {
             EMPTY_STRING
         } else {
-            let all = fields.join(COMMA);
+            let all = core_fields.join(COMMA);
             "&fl=".append(&all)
         }
     }