Rebuild pattern matching by parsing the bids schema, fix a small bug with repeated data_type in some filenames/paths

Author: gdevenyi

generate_entity_patterns.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+entities_order=($(jq -r .rules.entities.[] schema.json))
+entities_names_ordered=()
+
+for entity in ${entities_order[@]}; do
+  entity_name=$(jq -r .objects.entities.${entity}.name schema.json)
+  entities_names_ordered+=(${entity_name})
+  entity_format=$(jq -r .objects.entities.${entity}.format schema.json)
+  if [[ ${entity_format} == "label" ]]; then
+    echo \""*(_${entity_name}-+([a-zA-Z0-9]))"\"
+  elif [[ ${entity_format} == "index" ]]; then
+    echo \""*(_${entity_name}-+([0-9]))"\"
+  else
+    echo "Unrecognized entity_format ${entity_format}" 1>&2
+    exit 1
+  fi
+done
+
+printf "%s," ${entities_order[@]}
+echo
+printf "%s " ${entities_names_ordered[@]}
+echo

libBIDS.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-if (( BASH_VERSINFO[0] < 4 || (BASH_VERSINFO[0] == 4 && BASH_VERSINFO[1] < 3) )); then
+if ((BASH_VERSINFO[0] < 4 || (BASH_VERSINFO[0] == 4 && BASH_VERSINFO[1] < 3))); then
   echo "Error: bash >= 4.3 is required for this script" >&2
   exit 1
 fi
@@ -220,7 +220,9 @@ _libBIDSsh_parse_filename() {
   # Store the full path and filename
   arr[path]=$(tr -s / <<<"${path}")
   arr[extension]="${filename#*.}"
-  arr[type]=$(grep -E -o '(func|dwi|fmap|anat|perf|meg|eeg|ieeg|beh|pet|micr|nirs|motion|mrs)' <<<$(basename $(dirname "${path}")) || echo "NA")
+  # Extract from schema
+  # jq -r .objects.datatypes.[].value schema.json  | paste -s -d'|'
+  arr[data_type]=$(grep -o -E "(anat|beh|dwi|eeg|fmap|func|ieeg|meg|micr|motion|mrs|perf|pet|nirs)" <<<$(dirname ${path}) | head -1 || echo "NA")
   arr[derivatives]=$(grep -o 'derivatives/.*/' <<<"${path}" | awk -F/ '{print $2}' || echo "NA")
 
   local name_no_ext="${filename%%.*}"
@@ -245,13 +247,12 @@ _libBIDSsh_parse_filename() {
 
   key_order+=("suffix")
   key_order+=("extension")
-  key_order+=("type")
+  key_order+=("data_type")
   key_order+=("derivatives")
   key_order+=("path")
 
   # Store the key order in the array
   arr[_key_order]="${key_order[*]}"
-
 }
 
 libBIDSsh_extension_json_rows_to_column_json_path() {
@@ -337,49 +338,54 @@ libBIDSsh_parse_bids_to_csv() {
   # Build the pattern piece by piece
   local base_pattern="*"
 
-  # Optional components
-  local optional_components=(
-    "*(_ses-+([a-zA-Z0-9]))"          # _ses-<label>
-    "*(_task-+([a-zA-Z0-9]))"         # _task-<label>
-    "*(_acq-+([a-zA-Z0-9]))"          # _acq-<label>
-    "*(_ce-+([a-zA-Z0-9]))"           # _ce-<label>
-    "*(_rec-+([a-zA-Z0-9]))"          # _rec-<label>
-    "*(_dir-+([a-zA-Z0-9]))"          # _dir-<label>
-    "*(_run-+([0-9]))"                # _run-<index>
-    "*(_recording-+([a-zA-Z0-9]))"    # _recording-<label>
-    "*(_mod-+([a-zA-Z0-9]))"          # _mod-<label>
-    "*(_echo-+([0-9]))"               # _echo-<index>
-    "*(_part-@(mag|phase|real|imag))" # _part-<mag|phase|real|imag>
-    "*(_chunk-+([0-9]))"              # _chunk-<index>
+  # Entities components
+  # Extracted from schema with generate_entity_patterns.sh
+  local entities=(
+    "*(_sub-+([a-zA-Z0-9]))"
+    "*(_ses-+([a-zA-Z0-9]))"
+    "*(_sample-+([a-zA-Z0-9]))"
+    "*(_task-+([a-zA-Z0-9]))"
+    "*(_tracksys-+([a-zA-Z0-9]))"
+    "*(_acq-+([a-zA-Z0-9]))"
+    "*(_nuc-+([a-zA-Z0-9]))"
+    "*(_voi-+([a-zA-Z0-9]))"
+    "*(_ce-+([a-zA-Z0-9]))"
+    "*(_trc-+([a-zA-Z0-9]))"
+    "*(_stain-+([a-zA-Z0-9]))"
+    "*(_rec-+([a-zA-Z0-9]))"
+    "*(_dir-+([a-zA-Z0-9]))"
+    "*(_run-+([0-9]))"
+    "*(_mod-+([a-zA-Z0-9]))"
+    "*(_echo-+([0-9]))"
+    "*(_flip-+([0-9]))"
+    "*(_inv-+([0-9]))"
+    "*(_mt-+([a-zA-Z0-9]))"
+    "*(_part-+([a-zA-Z0-9]))"
+    "*(_proc-+([a-zA-Z0-9]))"
+    "*(_hemi-+([a-zA-Z0-9]))"
+    "*(_space-+([a-zA-Z0-9]))"
+    "*(_split-+([0-9]))"
+    "*(_recording-+([a-zA-Z0-9]))"
+    "*(_chunk-+([0-9]))"
+    "*(_seg-+([a-zA-Z0-9]))"
+    "*(_res-+([a-zA-Z0-9]))"
+    "*(_den-+([a-zA-Z0-9]))"
+    "*(_label-+([a-zA-Z0-9]))"
+    "*(_desc-+([a-zA-Z0-9]))"
   )
 
-  # Anatomical suffixes
-  # UNIT1 suffix is technically incorrect as it cannot coexist with the _part-@(mag|phase|real|imag)
-  local suffixes="_@(FLAIR|PDT2|PDw|T1w|T2starw|T2w|UNIT1|angio|inplaneT1|inplaneT2"
-  # Parametric map suffixes
-  suffixes+="|Chimap|M0map|MTRmap|MTVmap|MTsat|MWFmap|PDmap|R1map|R2map|R2starmap|RB1map|S0map|T1map|T1rho|T2map|T2starmap|TB1map"
-  # Defacing mask
-  suffixes+="|defacemask"
-  # Depreciated anatomical suffixes
-  # FLASH PD and T2star are depreciated but we support them
-  suffixes+="|FLASH|PD|T2star"
-  # Functional images
-  suffixes+="|bold|cbv|phase|sbref|noRF|events|physio|stim"
-  # Diffusion images
-  suffixes+="|dwi|sbref"
-  # Perfusion images
-  suffixes+="|asl|m0scan|aslcontext|noRF"
-  # Field maps
-  suffixes+="|magnitude1|magnitude2|phasediff|phase1|phase2|fieldmap|magnitude|epi"
-  # Modality agnostic files
-  suffixes+="|scans|sessions)"
+  # Suffixes from schema.json
+  # jq -r .objects.suffixes.[].value schema.json | paste -s -d'|'
+  suffixes+="_@(2PE|BF|Chimap|CARS|CONF|DIC|DF|FLAIR|FLASH|FLUO|IRT1|M0map|MEGRE|MESE|MP2RAGE|MPE|MPM|MTR|MTRmap|MTS|MTVmap|MTsat|MWFmap|NLO|OCT|PC|PD|PDT2|PDmap|PDw|PLI|R1map|R2map|R2starmap|RB1COR|RB1map|S0map|SEM|SPIM|SR|T1map|T1rho|T1w|T2map|T2star|T2starmap|T2starw|T2w|TB1AFI|TB1DAM|TB1EPI|TB1RFM|TB1SRGE|TB1TFL|TB1map|TEM|UNIT1|VFA|angio|asl|aslcontext|asllabeling|beh|blood|bold|cbv|channels|coordsystem|defacemask|descriptions|dseg|dwi|eeg|electrodes|epi|events|fieldmap|headshape|XPCT|ieeg|inplaneT1|inplaneT2|m0scan|magnitude|magnitude1|magnitude2|markers|mask|meg|motion|mrsi|mrsref|nirs|noRF|optodes|pet|phase|phase1|phase2|phasediff|photo|physio|probseg|sbref|scans|sessions|stim|svs|uCT|unloc)"
 
   # Allowed extensions
-  local extensions="@(.nii|.json|.tsv|bval|bvec)?(.gz)"
+  # jq -r .objects.extensions.[].value schema.json | paste -s -d'|'
+  # Stripped the ".*" and directory entries manually
+  local extensions="@(.ave|.bdf|.bval|.bvec|.chn|.con|.dat|.dlabel.nii|.edf|.eeg|.fdt|.fif|.jpg|.json|.kdf|.label.gii|.md||.mhd|.mrk|.nii|.nii.gz|.nwb|.ome.btf|.ome.tif|.png|.pos|.raw|.rst|.set|.snirf|.sqd|.tif|.trg|.tsv|.tsv.gz|.txt|.vhdr|.vmrk)"
 
   # Piece together the pattern
   local pattern=${base_pattern}
-  for entry in "${optional_components[@]}"; do
+  for entry in "${entities[@]}"; do
     pattern+=${entry}
   done
   pattern+=${suffixes}
@@ -395,11 +401,15 @@ libBIDSsh_parse_bids_to_csv() {
   shopt -u nullglob
   shopt -u globstar
 
-  echo "derivatives,sub,ses,type,task,acq,ce,rec,dir,run,recording,mod,echo,part,chunk,suffix,extension,path"
+  # Order of entities from generate_entity_patterns.sh
+  entities_displayname_order="subject,session,sample,task,tracksys,acquisition,nucleus,volume,ceagent,tracer,stain,reconstruction,direction,run,modality,echo,flip,inversion,mtransfer,part,processing,hemisphere,space,split,recording,chunk,segmentation,resolution,density,label,description"
+  entities_order="sub ses sample task tracksys acq nuc voi ce trc stain rec dir run mod echo flip inv mt part proc hemi space split recording chunk seg res den label desc"
+
+  echo "derivatives,data_type,${entities_displayname_order},suffix,extension,path"
   for file in "${files[@]}"; do
     declare -A file_info
     _libBIDSsh_parse_filename "${file}" file_info
-    for key in derivatives sub ses type task acq ce rec dir run recording mod echo part chunk suffix extension path; do
+    for key in derivatives data_type ${entities_order} suffix extension path; do
       if [[ "${file_info[${key}]+abc}" ]]; then
         echo -n "${file_info[${key}]},"
       else
@@ -617,16 +627,16 @@ libBIDS_csv_iterator() {
 }
 
 libBIDSsh_json_to_associative_array() {
-    local json_file="$1"
-    declare -n arr_ref="$2"  # nameref to the associative array
-
-    # Use jq to process the JSON file and output key-value pairs with type prefixes
-    while IFS="=" read -r key value; do
-        # Remove quotes from key (jq outputs keys with quotes)
-        key="${key%\"}"
-        key="${key#\"}"
-        arr_ref["$key"]="$value"
-    done < <(jq -r 'to_entries[] |
+  local json_file="$1"
+  declare -n arr_ref="$2" # nameref to the associative array
+
+  # Use jq to process the JSON file and output key-value pairs with type prefixes
+  while IFS="=" read -r key value; do
+    # Remove quotes from key (jq outputs keys with quotes)
+    key="${key%\"}"
+    key="${key#\"}"
+    arr_ref["$key"]="$value"
+  done < <(jq -r 'to_entries[] |
         "\(.key)=\(
             if .value|type == "array" then "array:" + (.value|join(","))
             elif .value|type == "object" then "object:" + (.value|tostring)