fix: improve separator detection and error handling in CSV processing

Author: dwpeng

src/filterx/src/files/csv.rs

@@ -1,9 +1,8 @@
 use crate::args::{CsvCommand, ShareArgs};
+use filterx_core::{util, writer::FilterxWriter, FilterxResult};
 use filterx_engine::vm::Vm;
 use filterx_source::{detect_columns, DataframeSource, Source, SourceType};
 
-use filterx_core::{util, writer::FilterxWriter, FilterxResult};
-
 pub fn filterx_csv(cmd: CsvCommand) -> FilterxResult<()> {
     let CsvCommand {
         share_args:
@@ -25,7 +24,13 @@ pub fn filterx_csv(cmd: CsvCommand) -> FilterxResult<()> {
     } = cmd;
     let separator = match separator {
         Some(s) => Some(s),
-        None => util::detect_separator(path.as_str(), 20)?,
+        None => match util::detect_separator(path.as_str(), 20)? {
+            Some(s) => Some(s),
+            None => {
+                eprintln!("Cannot detect separator, parse as one column.");
+                None
+            }
+        },
     };
     let output_separator;
     if _output_separator.is_none() {

src/filterx_core/src/util.rs

@@ -6,7 +6,8 @@ use polars::{
 };
 
 use crate::{
-    sep::Separator, thread_size::ThreadSize, writer::FilterxWriter, FilterxError, FilterxResult,
+    reader::FilterxReader, sep::Separator, thread_size::ThreadSize, writer::FilterxWriter,
+    FilterxError, FilterxResult,
 };
 use std::io::Write;
 use std::num::NonZero;
@@ -146,10 +147,9 @@ pub fn write_df(
 }
 
 pub fn collect_comment_lines(path: &str, comment_prefix: &str) -> FilterxResult<Vec<String>> {
-    use std::fs::File;
     use std::io::BufRead;
     use std::io::BufReader;
-    let file = File::open(path)?;
+    let file = FilterxReader::new(path)?;
     let mut reader = BufReader::new(file);
     let mut line = String::new();
     let mut comment_lines = Vec::new();
@@ -166,10 +166,9 @@ pub fn collect_comment_lines(path: &str, comment_prefix: &str) -> FilterxResult<
 }
 
 pub fn detect_separator(path: &str, nline: usize) -> FilterxResult<Option<String>> {
-    use std::fs::File;
     use std::io::BufRead;
     use std::io::BufReader;
-    let file = File::open(path)?;
+    let file = FilterxReader::new(path)?;
     let mut reader = BufReader::new(file);
     let mut line = String::new();
     let mut lines = Vec::with_capacity(nline);