Improve the log output, better path handling for Windows (#217)

Author: amd-arsuresh

mlc/index.py

@@ -40,6 +40,16 @@ def __init__(self, repos_path, repos):
         self._load_existing_index()
         self.build_index()
 
+    def _get_stored_mtime(self, key):
+        """
+        Helper method to safely extract mtime from stored data.
+        Handles both old format (direct mtime) and new format (dict with mtime key).
+        """
+        old = self.modified_times.get(key)
+        if old is None:
+            return None
+        return old["mtime"] if isinstance(old, dict) else old
+
     def _load_modified_times(self):
         """
         Load stored mtimes to check for changes in scripts.
@@ -49,7 +59,8 @@ def _load_modified_times(self):
                 # logger.info(f"Loading modified times from {self.modified_times_file}")
                 with open(self.modified_times_file, "r") as f:
                     return json.load(f)
-            except Exception:
+            except (json.JSONDecodeError, IOError) as e:
+                logger.warning(f"Failed to load modified times: {e}")
                 return {}
         return {}
 
@@ -76,7 +87,8 @@ def _load_existing_index(self):
                         if isinstance(item.get("repo"), dict):
                             item["repo"] = Repo(**item["repo"])
 
-                except Exception:
+                except (json.JSONDecodeError, IOError, KeyError, TypeError) as e:
+                    logger.warning(f"Failed to load index for {folder_type}: {e}")
                     pass   # fall back to empty index
 
     def add(self, meta, folder_type, path, repo):
@@ -165,25 +177,32 @@ def _index_single_repo(self, repo, repos_changed=False, current_item_keys=None):
                 elif os.path.isfile(json_path):
                     config_path = json_path
                 else:
-                    #logger.debug(f"No config file found in {automation_path}, skipping")
+                    # No config file found, remove from index if exists
                     delete_flag = False
-                    if automation_dir in self.modified_times:
-                        del self.modified_times[automation_dir]
-                    if any(automation_dir in item["path"] for item in self.indices[folder_type]):
+                    
+                    # Check and remove both possible config paths from modified_times
+                    for config_name in ["meta.yaml", "meta.json"]:
+                        config_key = os.path.join(automation_path, config_name)
+                        if config_key in self.modified_times:
+                            del self.modified_times[config_key]
+                            delete_flag = True
+                    
+                    # Use exact path matching instead of substring
+                    if any(item["path"] == automation_path for item in self.indices[folder_type]):
                         logger.debug(f"Removed index entry (if it exists) for {folder_type} : {automation_dir}")
                         delete_flag = True
                         self._remove_index_entry(automation_path)
+                    
                     if delete_flag:
                         self._save_indices()
                     continue
                 if current_item_keys is not None:
                     current_item_keys.add(config_path)
                 mtime = self.get_item_mtime(config_path)
-                old = self.modified_times.get(config_path)
-                old_mtime = old["mtime"] if isinstance(old, dict) else old
+                old_mtime = self._get_stored_mtime(config_path)
 
                 # skip if unchanged
-                if old_mtime == mtime and repos_changed != 1:
+                if old_mtime == mtime and not repos_changed:
                     continue
 
                 self.modified_times[config_path] = {
@@ -220,6 +239,7 @@ def build_index(self):
             self.modified_times = {}
             self.indices = {k: [] for k in self.index_files.keys()}
             force_rebuild = True
+           
 
         # index each repo
         for repo in self.repos:
@@ -246,10 +266,12 @@ def build_index(self):
 
     def _remove_index_entry(self, key):
         logger.debug(f"Removing index entry for {key}")
+        # Normalize paths for comparison
+        normalized_key = os.path.normpath(key)
         for ft in self.indices:
             self.indices[ft] = [
                 item for item in self.indices[ft]
-                if key not in item["path"]
+                if os.path.normpath(item["path"]) != normalized_key
             ]
 
     def _delete_by_uid(self, folder_type, uid, alias):
@@ -302,17 +324,14 @@ def _process_config_file(self, config_file, folder_type, folder_path, repo):
             alias = data.get("alias", None)
 
             # Validate and add to indices
-            if unique_id:
-                self._delete_by_uid(folder_type, unique_id, alias)
-                self.indices[folder_type].append({
-                    "uid": unique_id,
-                    "tags": tags,
-                    "alias": alias,
-                    "path": folder_path,
-                    "repo": repo
-                })
-            else:
-                logger.warning(f"Skipping {config_file}: Missing 'uid' field.")
+            self._delete_by_uid(folder_type, unique_id, alias)
+            self.indices[folder_type].append({
+                "uid": unique_id,
+                "tags": tags,
+                "alias": alias,
+                "path": folder_path,
+                "repo": repo
+            })
 
         except Exception as e:
             logger.error(f"Error processing {config_file}: {e}")

mlc/logger.py

@@ -15,17 +15,27 @@ class ColoredFormatter(logging.Formatter):
     }
 
     def format(self, record):
-        # Add color to the levelname
+        # Pad filename and line number for alignment
+        record.filename = f"{record.filename:<15}"  # Left-align filename with 15 char width
+        record.lineno = f"{record.lineno:>4}"  # Right-align line number with 4 char width
+        
+        # Trim WARNING to WARN
+        levelname = "WARN" if record.levelname == "WARNING" else record.levelname
+        
+        # Pad and add color to the levelname
+        levelname_padded = f"{levelname:<5}"  # Left-align levelname with 5 char width
         if record.levelname in self.COLORS:
-            record.levelname = f"{self.COLORS[record.levelname]}{record.levelname}{Style.RESET_ALL}"
+            record.levelname = f"{self.COLORS[record.levelname]}{levelname_padded}{Style.RESET_ALL}"
+        else:
+            record.levelname = levelname_padded
         return super().format(record)
 
 
 # Set up logging configuration
 def setup_logging(log_path = os.getcwd(), log_file = '.mlc-log.txt'):
 
     if not logger.hasHandlers():
-        logFormatter = ColoredFormatter('[%(asctime)s %(filename)s:%(lineno)d %(levelname)s] - %(message)s')
+        logFormatter = ColoredFormatter('[%(asctime)s %(filename)s:%(lineno)s %(levelname)s] - %(message)s')
         # by default logging level is set to INFO is being set
         logger.setLevel(logging.INFO)
 

mlc/utils.py

@@ -694,24 +694,30 @@ def extract_file(options):
         with zipfile.ZipFile(filename, 'r') as archive:
             members = archive.namelist()
             for member in members:
-                # Strip folder levels
-                stripped_path = os.path.join(
-                    extract_to, *member.split(os.sep)[strip_folders:]
-                )
-                if member.endswith('/'):  # Directory
-                    os.makedirs(stripped_path, exist_ok=True)
-                else:  # File
-                    os.makedirs(os.path.dirname(stripped_path), exist_ok=True)
-                    with archive.open(member) as source, open(stripped_path, 'wb') as target:
-                        shutil.copyfileobj(source, target)
+                # Strip folder levels (zip files always use forward slashes internally)
+                parts = member.split('/')
+                if len(parts) > strip_folders:
+                    stripped_parts = parts[strip_folders:]
+                    stripped_path = os.path.join(extract_to, *stripped_parts)
+                    stripped_path = os.path.normpath(stripped_path)
+                    
+                    if member.endswith('/'):  # Directory
+                        os.makedirs(stripped_path, exist_ok=True)
+                    else:  # File
+                        os.makedirs(os.path.dirname(stripped_path), exist_ok=True)
+                        with archive.open(member) as source, open(stripped_path, 'wb') as target:
+                            shutil.copyfileobj(source, target)
 
     elif tarfile.is_tarfile(filename):
         with tarfile.open(filename, 'r') as archive:
             members = archive.getmembers()
             for member in members:
                 if strip_folders:
+                    # Tar files also use forward slashes internally
                     parts = member.name.split('/')
-                    member.name = '/'.join(parts[strip_folders:])
+                    if len(parts) > strip_folders:
+                        # Join with OS-specific separator for extraction
+                        member.name = os.path.join(*parts[strip_folders:])
                 archive.extract(member, path=extract_to)
 
     else: