From c1c810a79aee768da189269ca1c78821348f15ea Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 14 Jan 2026 16:59:20 +0100 Subject: [PATCH 01/17] Misc fixes for processor plugins --- changedetectionio/async_update_worker.py | 13 +- changedetectionio/blueprint/ui/diff.py | 153 ++++++++++---------- changedetectionio/blueprint/ui/preview.py | 89 ++++++------ changedetectionio/pluggy_interface.py | 24 ++++ changedetectionio/processors/__init__.py | 165 +++++++++++++++++----- 5 files changed, 276 insertions(+), 168 deletions(-) diff --git a/changedetectionio/async_update_worker.py b/changedetectionio/async_update_worker.py index e8b8ebd5ae0..b7182623491 100644 --- a/changedetectionio/async_update_worker.py +++ b/changedetectionio/async_update_worker.py @@ -114,11 +114,14 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec processor = watch.get('processor', 'text_json_diff') # Init a new 'difference_detection_processor' - try: - processor_module = importlib.import_module(f"changedetectionio.processors.{processor}.processor") - except ModuleNotFoundError as e: - print(f"Processor module '{processor}' not found.") - raise e + # Use get_processor_module() to support both built-in and plugin processors + from changedetectionio.processors import get_processor_module + processor_module = get_processor_module(processor) + + if not processor_module: + error_msg = f"Processor module '{processor}' not found." + logger.error(error_msg) + raise ModuleNotFoundError(error_msg) update_handler = processor_module.perform_site_check(datastore=datastore, watch_uuid=uuid) diff --git a/changedetectionio/blueprint/ui/diff.py b/changedetectionio/blueprint/ui/diff.py index 7f30c57a84c..b83605642e7 100644 --- a/changedetectionio/blueprint/ui/diff.py +++ b/changedetectionio/blueprint/ui/diff.py @@ -101,23 +101,21 @@ def diff_history_page(uuid): # Get the processor type for this watch processor_name = watch.get('processor', 'text_json_diff') - try: - # Try to import the processor's difference module - processor_module = importlib.import_module(f'changedetectionio.processors.{processor_name}.difference') - - # Call the processor's render() function - if hasattr(processor_module, 'render'): - return processor_module.render( - watch=watch, - datastore=datastore, - request=request, - url_for=url_for, - render_template=render_template, - flash=flash, - redirect=redirect - ) - except (ImportError, ModuleNotFoundError) as e: - logger.warning(f"Processor {processor_name} does not have a difference module, falling back to text_json_diff: {e}") + # Try to get the processor's difference module (works for both built-in and plugin processors) + from changedetectionio.processors import get_processor_submodule + processor_module = get_processor_submodule(processor_name, 'difference') + + # Call the processor's render() function + if processor_module and hasattr(processor_module, 'render'): + return processor_module.render( + watch=watch, + datastore=datastore, + request=request, + url_for=url_for, + render_template=render_template, + flash=flash, + redirect=redirect + ) # Fallback: if processor doesn't have difference module, use text_json_diff as default from changedetectionio.processors.text_json_diff.difference import render as default_render @@ -157,23 +155,21 @@ def diff_history_page_extract_GET(uuid): # Get the processor type for this watch processor_name = watch.get('processor', 'text_json_diff') - try: - # Try to import the processor's extract module - processor_module = importlib.import_module(f'changedetectionio.processors.{processor_name}.extract') - - # Call the processor's render_form() function - if hasattr(processor_module, 'render_form'): - return processor_module.render_form( - watch=watch, - datastore=datastore, - request=request, - url_for=url_for, - render_template=render_template, - flash=flash, - redirect=redirect - ) - except (ImportError, ModuleNotFoundError) as e: - logger.warning(f"Processor {processor_name} does not have an extract module, falling back to base extractor: {e}") + # Try to get the processor's extract module (works for both built-in and plugin processors) + from changedetectionio.processors import get_processor_submodule + processor_module = get_processor_submodule(processor_name, 'extract') + + # Call the processor's render_form() function + if processor_module and hasattr(processor_module, 'render_form'): + return processor_module.render_form( + watch=watch, + datastore=datastore, + request=request, + url_for=url_for, + render_template=render_template, + flash=flash, + redirect=redirect + ) # Fallback: if processor doesn't have extract module, use base processors.extract as default from changedetectionio.processors.extract import render_form as default_render_form @@ -213,24 +209,22 @@ def diff_history_page_extract_POST(uuid): # Get the processor type for this watch processor_name = watch.get('processor', 'text_json_diff') - try: - # Try to import the processor's extract module - processor_module = importlib.import_module(f'changedetectionio.processors.{processor_name}.extract') - - # Call the processor's process_extraction() function - if hasattr(processor_module, 'process_extraction'): - return processor_module.process_extraction( - watch=watch, - datastore=datastore, - request=request, - url_for=url_for, - make_response=make_response, - send_from_directory=send_from_directory, - flash=flash, - redirect=redirect - ) - except (ImportError, ModuleNotFoundError) as e: - logger.warning(f"Processor {processor_name} does not have an extract module, falling back to base extractor: {e}") + # Try to get the processor's extract module (works for both built-in and plugin processors) + from changedetectionio.processors import get_processor_submodule + processor_module = get_processor_submodule(processor_name, 'extract') + + # Call the processor's process_extraction() function + if processor_module and hasattr(processor_module, 'process_extraction'): + return processor_module.process_extraction( + watch=watch, + datastore=datastore, + request=request, + url_for=url_for, + make_response=make_response, + send_from_directory=send_from_directory, + flash=flash, + redirect=redirect + ) # Fallback: if processor doesn't have extract module, use base processors.extract as default from changedetectionio.processors.extract import process_extraction as default_process_extraction @@ -280,38 +274,33 @@ def processor_asset(uuid, asset_name): # Get the processor type for this watch processor_name = watch.get('processor', 'text_json_diff') - try: - # Try to import the processor's difference module - processor_module = importlib.import_module(f'changedetectionio.processors.{processor_name}.difference') - - # Call the processor's get_asset() function - if hasattr(processor_module, 'get_asset'): - result = processor_module.get_asset( - asset_name=asset_name, - watch=watch, - datastore=datastore, - request=request - ) - - if result is None: - from flask import abort - abort(404, description=f"Asset '{asset_name}' not found") - - binary_data, content_type, cache_control = result - - response = make_response(binary_data) - response.headers['Content-Type'] = content_type - if cache_control: - response.headers['Cache-Control'] = cache_control - return response - else: - logger.warning(f"Processor {processor_name} does not implement get_asset()") + # Try to get the processor's difference module (works for both built-in and plugin processors) + from changedetectionio.processors import get_processor_submodule + processor_module = get_processor_submodule(processor_name, 'difference') + + # Call the processor's get_asset() function + if processor_module and hasattr(processor_module, 'get_asset'): + result = processor_module.get_asset( + asset_name=asset_name, + watch=watch, + datastore=datastore, + request=request + ) + + if result is None: from flask import abort - abort(404, description=f"Processor '{processor_name}' does not support assets") + abort(404, description=f"Asset '{asset_name}' not found") + + binary_data, content_type, cache_control = result - except (ImportError, ModuleNotFoundError) as e: - logger.warning(f"Processor {processor_name} does not have a difference module: {e}") + response = make_response(binary_data) + response.headers['Content-Type'] = content_type + if cache_control: + response.headers['Cache-Control'] = cache_control + return response + else: + logger.warning(f"Processor {processor_name} does not implement get_asset()") from flask import abort - abort(404, description=f"Processor '{processor_name}' not found") + abort(404, description=f"Processor '{processor_name}' does not support assets") return diff_blueprint diff --git a/changedetectionio/blueprint/ui/preview.py b/changedetectionio/blueprint/ui/preview.py index 9ad56371990..c735266a566 100644 --- a/changedetectionio/blueprint/ui/preview.py +++ b/changedetectionio/blueprint/ui/preview.py @@ -39,24 +39,21 @@ def preview_page(uuid): # Get the processor type for this watch processor_name = watch.get('processor', 'text_json_diff') - try: - # Try to import the processor's preview module - import importlib - processor_module = importlib.import_module(f'changedetectionio.processors.{processor_name}.preview') - - # Call the processor's render() function - if hasattr(processor_module, 'render'): - return processor_module.render( - watch=watch, - datastore=datastore, - request=request, - url_for=url_for, - render_template=render_template, - flash=flash, - redirect=redirect - ) - except (ImportError, ModuleNotFoundError) as e: - logger.debug(f"Processor {processor_name} does not have a preview module, using default preview: {e}") + # Try to get the processor's preview module (works for both built-in and plugin processors) + from changedetectionio.processors import get_processor_submodule + processor_module = get_processor_submodule(processor_name, 'preview') + + # Call the processor's render() function + if processor_module and hasattr(processor_module, 'render'): + return processor_module.render( + watch=watch, + datastore=datastore, + request=request, + url_for=url_for, + render_template=render_template, + flash=flash, + redirect=redirect + ) # Fallback: if processor doesn't have preview module, use default text preview content = [] @@ -163,39 +160,33 @@ def processor_asset(uuid, asset_name): # Get the processor type for this watch processor_name = watch.get('processor', 'text_json_diff') - try: - # Try to import the processor's preview module - import importlib - processor_module = importlib.import_module(f'changedetectionio.processors.{processor_name}.preview') - - # Call the processor's get_asset() function - if hasattr(processor_module, 'get_asset'): - result = processor_module.get_asset( - asset_name=asset_name, - watch=watch, - datastore=datastore, - request=request - ) - - if result is None: - from flask import abort - abort(404, description=f"Asset '{asset_name}' not found") - - binary_data, content_type, cache_control = result - - response = make_response(binary_data) - response.headers['Content-Type'] = content_type - if cache_control: - response.headers['Cache-Control'] = cache_control - return response - else: - logger.warning(f"Processor {processor_name} does not implement get_asset()") + # Try to get the processor's preview module (works for both built-in and plugin processors) + from changedetectionio.processors import get_processor_submodule + processor_module = get_processor_submodule(processor_name, 'preview') + + # Call the processor's get_asset() function + if processor_module and hasattr(processor_module, 'get_asset'): + result = processor_module.get_asset( + asset_name=asset_name, + watch=watch, + datastore=datastore, + request=request + ) + + if result is None: from flask import abort - abort(404, description=f"Processor '{processor_name}' does not support assets") + abort(404, description=f"Asset '{asset_name}' not found") + + binary_data, content_type, cache_control = result - except (ImportError, ModuleNotFoundError) as e: - logger.warning(f"Processor {processor_name} does not have a preview module: {e}") + response = make_response(binary_data) + response.headers['Content-Type'] = content_type + if cache_control: + response.headers['Cache-Control'] = cache_control + return response + else: + logger.warning(f"Processor {processor_name} does not implement get_asset()") from flask import abort - abort(404, description=f"Processor '{processor_name}' not found") + abort(404, description=f"Processor '{processor_name}' does not support assets") return preview_blueprint diff --git a/changedetectionio/pluggy_interface.py b/changedetectionio/pluggy_interface.py index 08dba213f6b..49647b14158 100644 --- a/changedetectionio/pluggy_interface.py +++ b/changedetectionio/pluggy_interface.py @@ -105,6 +105,30 @@ def plugin_settings_tab(self): """ pass + @hookspec + def register_processor(self): + """Register an external processor plugin. + + External packages can implement this hook to register custom processors + that will be discovered alongside built-in processors. + + Returns: + dict or None: Dictionary with processor information: + { + 'processor_name': str, # Machine name (e.g., 'osint_recon') + 'processor_module': module, # Module containing processor.py + 'processor_class': class, # The perform_site_check class + 'metadata': { # Optional metadata + 'name': str, # Display name + 'description': str, # Description + 'processor_weight': int,# Sort weight (lower = higher priority) + 'list_badge_text': str, # Badge text for UI + } + } + Return None if this plugin doesn't provide a processor + """ + pass + # Set up Plugin Manager plugin_manager = pluggy.PluginManager(PLUGIN_NAMESPACE) diff --git a/changedetectionio/processors/__init__.py b/changedetectionio/processors/__init__.py index 29828a04b83..bd658e88999 100644 --- a/changedetectionio/processors/__init__.py +++ b/changedetectionio/processors/__init__.py @@ -17,9 +17,11 @@ def find_sub_packages(package_name): return [name for _, name, is_pkg in pkgutil.iter_modules(package.__path__) if is_pkg] +@lru_cache(maxsize=1) def find_processors(): """ Find all subclasses of DifferenceDetectionProcessor in the specified package. + Results are cached to avoid repeated discovery. :param package_name: The name of the package to scan for processor modules. :return: A list of (module, class) tuples. @@ -46,6 +48,22 @@ def find_processors(): except (ModuleNotFoundError, ImportError) as e: logger.warning(f"Failed to import module {module_name}: {e} (find_processors())") + # Discover plugin processors via pluggy + try: + from changedetectionio.pluggy_interface import plugin_manager + plugin_results = plugin_manager.hook.register_processor() + + for result in plugin_results: + if result and isinstance(result, dict): + processor_module = result.get('processor_module') + processor_name = result.get('processor_name') + + if processor_module and processor_name: + processors.append((processor_module, processor_name)) + logger.info(f"Registered plugin processor: {processor_name}") + except Exception as e: + logger.warning(f"Error loading plugin processors: {e}") + return processors @@ -97,54 +115,137 @@ def find_processor_module(processor_name): return None +def get_processor_module(processor_name): + """ + Get the actual processor module (with perform_site_check class) by name. + Works for both built-in and plugin processors. + + Args: + processor_name: Processor machine name (e.g., 'text_json_diff', 'osint_recon') + + Returns: + module: The processor module containing perform_site_check, or None if not found + """ + processor_classes = find_processors() + processor_tuple = next((tpl for tpl in processor_classes if tpl[1] == processor_name), None) + + if processor_tuple: + # Return the actual processor module (first element of tuple) + return processor_tuple[0] + + return None + + +def get_processor_submodule(processor_name, submodule_name): + """ + Get an optional submodule from a processor (e.g., 'difference', 'extract', 'preview'). + Works for both built-in and plugin processors. + + Args: + processor_name: Processor machine name (e.g., 'text_json_diff', 'osint_recon') + submodule_name: Name of the submodule (e.g., 'difference', 'extract', 'preview') + + Returns: + module: The submodule if it exists, or None if not found + """ + processor_classes = find_processors() + processor_tuple = next((tpl for tpl in processor_classes if tpl[1] == processor_name), None) + + if not processor_tuple: + return None + + processor_module = processor_tuple[0] + parent_module = get_parent_module(processor_module) + + if not parent_module: + return None + + # Try to import the submodule + try: + # For built-in processors: changedetectionio.processors.text_json_diff.difference + # For plugin processors: changedetectionio_osint.difference + parent_module_name = parent_module.__name__ + submodule_full_name = f"{parent_module_name}.{submodule_name}" + return importlib.import_module(submodule_full_name) + except (ModuleNotFoundError, ImportError): + return None + + +@lru_cache(maxsize=1) +def get_plugin_processor_metadata(): + """Get metadata from plugin processors.""" + metadata = {} + try: + from changedetectionio.pluggy_interface import plugin_manager + plugin_results = plugin_manager.hook.register_processor() + + for result in plugin_results: + if result and isinstance(result, dict): + processor_name = result.get('processor_name') + meta = result.get('metadata', {}) + if processor_name: + metadata[processor_name] = meta + except Exception as e: + logger.warning(f"Error getting plugin processor metadata: {e}") + return metadata + + def available_processors(): """ Get a list of processors by name and description for the UI elements. - Can be filtered via ALLOWED_PROCESSORS environment variable (comma-separated list). + Can be filtered via DISABLED_PROCESSORS environment variable (comma-separated list). :return: A list :) """ processor_classes = find_processors() - # Check if ALLOWED_PROCESSORS env var is set - # For now we disable it, need to make a deploy with lots of new code and this will be an overload - allowed_processors_env = os.getenv('ALLOWED_PROCESSORS', 'text_json_diff, restock_diff').strip() - allowed_processors = None - if allowed_processors_env: + # Check if DISABLED_PROCESSORS env var is set + disabled_processors_env = os.getenv('DISABLED_PROCESSORS', 'image_ssim_diff').strip() + disabled_processors = [] + if disabled_processors_env: # Parse comma-separated list and strip whitespace - allowed_processors = [p.strip() for p in allowed_processors_env.split(',') if p.strip()] - logger.info(f"ALLOWED_PROCESSORS set, filtering to: {allowed_processors}") + disabled_processors = [p.strip() for p in disabled_processors_env.split(',') if p.strip()] + logger.info(f"DISABLED_PROCESSORS set, disabling: {disabled_processors}") available = [] + plugin_metadata = get_plugin_processor_metadata() + for module, sub_package_name in processor_classes: - # Filter by allowed processors if set - if allowed_processors and sub_package_name not in allowed_processors: - logger.debug(f"Skipping processor '{sub_package_name}' (not in ALLOWED_PROCESSORS)") + # Skip disabled processors + if sub_package_name in disabled_processors: + logger.debug(f"Skipping processor '{sub_package_name}' (in DISABLED_PROCESSORS)") continue - # Try to get the 'name' attribute from the processor module first - if hasattr(module, 'name'): - description = gettext(module.name) + # Check if this is a plugin processor + if sub_package_name in plugin_metadata: + meta = plugin_metadata[sub_package_name] + description = gettext(meta.get('name', sub_package_name)) + # Plugin processors start from weight 10 to separate them from built-in processors + weight = 100 + meta.get('processor_weight', 0) else: - # Fall back to processor_description from parent module's __init__.py - parent_module = get_parent_module(module) - if parent_module and hasattr(parent_module, 'processor_description'): - description = gettext(parent_module.processor_description) + # Try to get the 'name' attribute from the processor module first + if hasattr(module, 'name'): + description = gettext(module.name) else: - # Final fallback to a readable name - description = sub_package_name.replace('_', ' ').title() - - # Get weight for sorting (lower weight = higher in list) - weight = 0 # Default weight for processors without explicit weight - - # Check processor module itself first - if hasattr(module, 'processor_weight'): - weight = module.processor_weight - else: - # Fall back to parent module (package __init__.py) - parent_module = get_parent_module(module) - if parent_module and hasattr(parent_module, 'processor_weight'): - weight = parent_module.processor_weight + # Fall back to processor_description from parent module's __init__.py + parent_module = get_parent_module(module) + if parent_module and hasattr(parent_module, 'processor_description'): + description = gettext(parent_module.processor_description) + else: + # Final fallback to a readable name + description = sub_package_name.replace('_', ' ').title() + + # Get weight for sorting (lower weight = higher in list) + weight = 0 # Default weight for processors without explicit weight + + # Check processor module itself first + if hasattr(module, 'processor_weight'): + weight = module.processor_weight + else: + # Fall back to parent module (package __init__.py) + parent_module = get_parent_module(module) + if parent_module and hasattr(parent_module, 'processor_weight'): + weight = parent_module.processor_weight available.append((sub_package_name, description, weight)) From 423b54694841ff986bcecfb1828fa381b1441a86 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 14 Jan 2026 17:29:20 +0100 Subject: [PATCH 02/17] Refactor of processor extra configs --- changedetectionio/api/Watch.py | 58 +++---------------- changedetectionio/blueprint/ui/edit.py | 56 ++---------------- changedetectionio/processors/__init__.py | 73 ++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 101 deletions(-) diff --git a/changedetectionio/api/Watch.py b/changedetectionio/api/Watch.py index d3e790e5304..784468974eb 100644 --- a/changedetectionio/api/Watch.py +++ b/changedetectionio/api/Watch.py @@ -139,58 +139,18 @@ def put(self, uuid): # Handle processor-config-* fields separately (save to JSON, not datastore) from changedetectionio import processors - processor_config_data = {} - regular_data = {} - - for key, value in request.json.items(): - if key.startswith('processor_config_'): - config_key = key.replace('processor_config_', '') - if value: # Only save non-empty values - processor_config_data[config_key] = value - else: - regular_data[key] = value + + # Make a mutable copy of request.json for modification + json_data = dict(request.json) + + # Extract and remove processor config fields from json_data + processor_config_data = processors.extract_processor_config_from_form_data(json_data) # Update watch with regular (non-processor-config) fields - watch.update(regular_data) + watch.update(json_data) - # Save processor config to JSON file if any config data exists - if processor_config_data: - try: - processor_name = request.json.get('processor', watch.get('processor')) - if processor_name: - # Create a processor instance to access config methods - from changedetectionio.processors import difference_detection_processor - processor_instance = difference_detection_processor(self.datastore, uuid) - # Use processor name as filename so each processor keeps its own config - config_filename = f'{processor_name}.json' - processor_instance.update_extra_watch_config(config_filename, processor_config_data) - logger.debug(f"API: Saved processor config to {config_filename}: {processor_config_data}") - - # Call optional edit_hook if processor has one - try: - import importlib - edit_hook_module_name = f'changedetectionio.processors.{processor_name}.edit_hook' - - try: - edit_hook = importlib.import_module(edit_hook_module_name) - logger.debug(f"API: Found edit_hook module for {processor_name}") - - if hasattr(edit_hook, 'on_config_save'): - logger.info(f"API: Calling edit_hook.on_config_save for {processor_name}") - # Call hook and get updated config - updated_config = edit_hook.on_config_save(watch, processor_config_data, self.datastore) - # Save updated config back to file - processor_instance.update_extra_watch_config(config_filename, updated_config) - logger.info(f"API: Edit hook updated config: {updated_config}") - else: - logger.debug(f"API: Edit hook module found but no on_config_save function") - except ModuleNotFoundError: - logger.debug(f"API: No edit_hook module for processor {processor_name} (this is normal)") - except Exception as hook_error: - logger.error(f"API: Edit hook error (non-fatal): {hook_error}", exc_info=True) - - except Exception as e: - logger.error(f"API: Failed to save processor config: {e}") + # Save processor config to JSON file + processors.save_processor_config(self.datastore, uuid, processor_config_data) return "OK", 200 diff --git a/changedetectionio/blueprint/ui/edit.py b/changedetectionio/blueprint/ui/edit.py index e2edd557b16..6e1c59be919 100644 --- a/changedetectionio/blueprint/ui/edit.py +++ b/changedetectionio/blueprint/ui/edit.py @@ -144,58 +144,10 @@ def edit_page(uuid): extra_update_obj['time_between_check'] = form.time_between_check.data - # Handle processor-config-* fields separately (save to JSON, not datastore) - processor_config_data = {} - fields_to_remove = [] - for field_name, field_value in form.data.items(): - if field_name.startswith('processor_config_'): - config_key = field_name.replace('processor_config_', '') - if field_value: # Only save non-empty values - processor_config_data[config_key] = field_value - fields_to_remove.append(field_name) - - # Save processor config to JSON file if any config data exists - if processor_config_data: - try: - processor_name = form.data.get('processor') - # Create a processor instance to access config methods - processor_instance = processors.difference_detection_processor(datastore, uuid) - # Use processor name as filename so each processor keeps its own config - config_filename = f'{processor_name}.json' - processor_instance.update_extra_watch_config(config_filename, processor_config_data) - logger.debug(f"Saved processor config to {config_filename}: {processor_config_data}") - - # Call optional edit_hook if processor has one - try: - # Try to import the edit_hook module from the processor package - import importlib - edit_hook_module_name = f'changedetectionio.processors.{processor_name}.edit_hook' - - try: - edit_hook = importlib.import_module(edit_hook_module_name) - logger.debug(f"Found edit_hook module for {processor_name}") - - if hasattr(edit_hook, 'on_config_save'): - logger.info(f"Calling edit_hook.on_config_save for {processor_name}") - watch_obj = datastore.data['watching'][uuid] - # Call hook and get updated config - updated_config = edit_hook.on_config_save(watch_obj, processor_config_data, datastore) - # Save updated config back to file - processor_instance.update_extra_watch_config(config_filename, updated_config) - logger.info(f"Edit hook updated config: {updated_config}") - else: - logger.debug(f"Edit hook module found but no on_config_save function") - except ModuleNotFoundError: - logger.debug(f"No edit_hook module for processor {processor_name} (this is normal)") - except Exception as hook_error: - logger.error(f"Edit hook error (non-fatal): {hook_error}", exc_info=True) - - except Exception as e: - logger.error(f"Failed to save processor config: {e}") - - # Remove processor-config-* fields from form.data before updating datastore - for field_name in fields_to_remove: - form.data.pop(field_name, None) + # Handle processor-config-* fields separately (save to JSON, not datastore) + # IMPORTANT: These must NOT be saved to url-watches.json, only to the processor-specific JSON file + processor_config_data = processors.extract_processor_config_from_form_data(form.data) + processors.save_processor_config(datastore, uuid, processor_config_data) # Ignore text form_ignore_text = form.ignore_text.data diff --git a/changedetectionio/processors/__init__.py b/changedetectionio/processors/__init__.py index bd658e88999..9cb9332c116 100644 --- a/changedetectionio/processors/__init__.py +++ b/changedetectionio/processors/__init__.py @@ -380,3 +380,76 @@ def get_processor_badge_css(): return '\n\n'.join(css_rules) + +def save_processor_config(datastore, watch_uuid, config_data): + """ + Save processor-specific configuration to JSON file. + + This is a shared helper function used by both the UI edit form and API endpoints + to consistently handle processor configuration storage. + + Args: + datastore: The application datastore instance + watch_uuid: UUID of the watch + config_data: Dictionary of configuration data to save (with processor_config_* prefix removed) + + Returns: + bool: True if saved successfully, False otherwise + """ + if not config_data: + return True + + try: + from changedetectionio.processors.base import difference_detection_processor + + # Get processor name from watch + watch = datastore.data['watching'].get(watch_uuid) + if not watch: + logger.error(f"Cannot save processor config: watch {watch_uuid} not found") + return False + + processor_name = watch.get('processor', 'text_json_diff') + + # Create a processor instance to access config methods + processor_instance = difference_detection_processor(datastore, watch_uuid) + + # Use processor name as filename so each processor keeps its own config + config_filename = f'{processor_name}.json' + processor_instance.update_extra_watch_config(config_filename, config_data) + + logger.debug(f"Saved processor config to {config_filename}: {config_data}") + return True + + except Exception as e: + logger.error(f"Failed to save processor config: {e}") + return False + + +def extract_processor_config_from_form_data(form_data): + """ + Extract processor_config_* fields from form data and return separate dicts. + + This is a shared helper function used by both the UI edit form and API endpoints + to consistently handle processor configuration extraction. + + IMPORTANT: This function modifies form_data in-place by removing processor_config_* fields. + + Args: + form_data: Dictionary of form data (will be modified in-place) + + Returns: + dict: Dictionary of processor config data (with processor_config_* prefix removed) + """ + processor_config_data = {} + + # Use list() to create a copy of keys since we're modifying the dict + for field_name in list(form_data.keys()): + if field_name.startswith('processor_config_'): + config_key = field_name.replace('processor_config_', '') + # Save all values (including empty strings) to allow explicit clearing of settings + processor_config_data[config_key] = form_data[field_name] + # Remove from form_data to prevent it from reaching datastore + del form_data[field_name] + + return processor_config_data + From edf0989cd4249bf03a9d078105fb47a003e7e87c Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 14 Jan 2026 17:34:54 +0100 Subject: [PATCH 03/17] Put the type badge first in the list --- .../watchlist/templates/watch-overview.html | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/changedetectionio/blueprint/watchlist/templates/watch-overview.html b/changedetectionio/blueprint/watchlist/templates/watch-overview.html index fa51302fea4..525e1368099 100644 --- a/changedetectionio/blueprint/watchlist/templates/watch-overview.html +++ b/changedetectionio/blueprint/watchlist/templates/watch-overview.html @@ -200,23 +200,24 @@ {% endif %}
- - {% if system_use_url_watchlist or watch.get('use_page_title_in_list') %} - {{ watch.label }} - {% else %} - {{ watch.get('title') or watch.link }} - {% endif %} -   - + {%- if watch['processor'] and watch['processor'] in processor_badge_texts -%} + {{ processor_badge_texts[watch['processor']] }} + {%- endif -%} + + {% if system_use_url_watchlist or watch.get('use_page_title_in_list') %} + {{ watch.label }} + {% else %} + {{ watch.get('title') or watch.link }} + {% endif %} +   + {%- if watch['processor'] == 'text_json_diff' -%} {%- if watch['has_ldjson_price_data'] and not watch['track_ldjson_price_data'] -%}
Switch to Restock & Price watch mode? Yes No
{%- endif -%} {%- endif -%} - {%- if watch['processor'] and watch['processor'] in processor_badge_texts -%} - {{ processor_badge_texts[watch['processor']] }} - {%- endif -%} + {%- for watch_tag_uuid, watch_tag in datastore.get_all_tags_for_watch(watch['uuid']).items() -%} {{ watch_tag.title }} {%- endfor -%} From 4dc5301de42c2784ddf119128b4fb533171ca9e2 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 14 Jan 2026 18:22:23 +0100 Subject: [PATCH 04/17] Downgrade for osint plugin and handle missing processors better --- changedetectionio/blueprint/ui/edit.py | 9 +++++++-- requirements.txt | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/changedetectionio/blueprint/ui/edit.py b/changedetectionio/blueprint/ui/edit.py index 6e1c59be919..53f93a1a13c 100644 --- a/changedetectionio/blueprint/ui/edit.py +++ b/changedetectionio/blueprint/ui/edit.py @@ -66,8 +66,13 @@ def edit_page(uuid): processor_name = datastore.data['watching'][uuid].get('processor', '') processor_classes = next((tpl for tpl in processors.find_processors() if tpl[1] == processor_name), None) if not processor_classes: - flash(gettext("Cannot load the edit form for processor/plugin '{}', plugin missing?").format(processor_classes[1]), 'error') - return redirect(url_for('watchlist.index')) + flash(gettext("Could not load '{}' processor, processor plugin might be missing. Please select a different processor.").format(processor_name), 'error') + # Fall back to default processor so user can still edit and change processor + processor_classes = next((tpl for tpl in processors.find_processors() if tpl[1] == 'text_json_diff'), None) + if not processor_classes: + # If even text_json_diff is missing, something is very wrong + flash(gettext("Could not load '{}' processor, processor plugin might be missing.").format(processor_name), 'error') + return redirect(url_for('watchlist.index')) parent_module = processors.get_parent_module(processor_classes[0]) diff --git a/requirements.txt b/requirements.txt index 013150063dd..d2fbfa4ea0c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -51,9 +51,9 @@ linkify-it-py # - Needed for apprise/spush, and maybe others? hopefully doesnt trigger a rust compile. # - Requires extra wheel for rPi, adds build time for arm/v8 which is not in piwheels -# Pinned to 43.0.1 for ARM compatibility (45.x may not have pre-built ARM wheels) +# Pinned to 44.x for ARM compatibility and sslyze compatibility (sslyze requires <45) # Also pinned because dependabot wants specific versions -cryptography==46.0.3 +cryptography==44.0.0 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315 # use any version other than 2.0.x due to https://github.com/eclipse/paho.mqtt.python/issues/814 From 7e853a4b461a6e0f5be4e557b3d6d43b49700a21 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 14 Jan 2026 19:01:30 +0100 Subject: [PATCH 05/17] Configurable paackages --- Dockerfile | 9 +++++++++ docker-compose.yml | 7 +++++++ docker-entrypoint.sh | 28 ++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+) create mode 100755 docker-entrypoint.sh diff --git a/Dockerfile b/Dockerfile index 8598d8ccb68..f7a399157c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -132,6 +132,15 @@ ENV LOGGER_LEVEL="$LOGGER_LEVEL" ENV LC_ALL=en_US.UTF-8 WORKDIR /app + +# Copy and set up entrypoint script for installing extra packages +COPY docker-entrypoint.sh /docker-entrypoint.sh +RUN chmod +x /docker-entrypoint.sh + +# Set entrypoint to handle EXTRA_PACKAGES env var +ENTRYPOINT ["/docker-entrypoint.sh"] + +# Default command (can be overridden in docker-compose.yml) CMD ["python", "./changedetection.py", "-d", "/datastore"] diff --git a/docker-compose.yml b/docker-compose.yml index 9d353dd8e8d..834ca9c2bb9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,13 @@ services: # Log output levels: TRACE, DEBUG(default), INFO, SUCCESS, WARNING, ERROR, CRITICAL # - LOGGER_LEVEL=TRACE # + # Install additional Python packages (processor plugins, etc.) + # Packages are installed at container startup and cached to avoid reinstalling on every restart + # Example: Install the OSINT reconnaissance processor plugin + # - EXTRA_PACKAGES=changedetection-osint-processor + # Multiple packages can be installed by separating with spaces: + # - EXTRA_PACKAGES=changedetection-osint-processor another-plugin + # # # Uncomment below and the "sockpuppetbrowser" to use a real Chrome browser (It uses the "playwright" protocol) # - PLAYWRIGHT_DRIVER_URL=ws://browser-sockpuppet-chrome:3000 diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 00000000000..ac243289c84 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -e + +# Install additional packages from EXTRA_PACKAGES env var +# Uses a marker file to avoid reinstalling on every container restart +INSTALLED_MARKER="/datastore/.extra_packages_installed" +CURRENT_PACKAGES="$EXTRA_PACKAGES" + +if [ -n "$EXTRA_PACKAGES" ]; then + # Check if we need to install/update packages + if [ ! -f "$INSTALLED_MARKER" ] || [ "$(cat $INSTALLED_MARKER 2>/dev/null)" != "$CURRENT_PACKAGES" ]; then + echo "Installing extra packages: $EXTRA_PACKAGES" + pip3 install --no-cache-dir $EXTRA_PACKAGES + + if [ $? -eq 0 ]; then + echo "$CURRENT_PACKAGES" > "$INSTALLED_MARKER" + echo "Extra packages installed successfully" + else + echo "ERROR: Failed to install extra packages" + exit 1 + fi + else + echo "Extra packages already installed: $EXTRA_PACKAGES" + fi +fi + +# Execute the main command +exec "$@" From b826d9b236cd3d2626877085694234bc18964865 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 3 Feb 2026 13:51:15 +0100 Subject: [PATCH 06/17] Bump docs --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 834ca9c2bb9..9979015986b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,9 +19,9 @@ services: # Install additional Python packages (processor plugins, etc.) # Packages are installed at container startup and cached to avoid reinstalling on every restart # Example: Install the OSINT reconnaissance processor plugin - # - EXTRA_PACKAGES=changedetection-osint-processor + # - EXTRA_PACKAGES=changedetection.io-osint # Multiple packages can be installed by separating with spaces: - # - EXTRA_PACKAGES=changedetection-osint-processor another-plugin + # - EXTRA_PACKAGES=changedetection.io-osint another-plugin # # # Uncomment below and the "sockpuppetbrowser" to use a real Chrome browser (It uses the "playwright" protocol) From e958acebed6d187143236e85cb297808b9c523a8 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 3 Feb 2026 15:31:44 +0100 Subject: [PATCH 07/17] Processors can specify what capabilities they support (visual selector, text filters etc) --- changedetectionio/blueprint/ui/edit.py | 6 ++++++ .../blueprint/ui/templates/edit.html | 18 ++++++++++++------ .../processors/image_ssim_diff/__init__.py | 6 ++++++ .../processors/text_json_diff/__init__.py | 7 ++++++- 4 files changed, 30 insertions(+), 7 deletions(-) diff --git a/changedetectionio/blueprint/ui/edit.py b/changedetectionio/blueprint/ui/edit.py index df8f32e9482..f97aae02fc7 100644 --- a/changedetectionio/blueprint/ui/edit.py +++ b/changedetectionio/blueprint/ui/edit.py @@ -267,6 +267,12 @@ def edit_page(uuid): # Get fetcher capabilities instead of hardcoded logic capabilities = get_fetcher_capabilities(watch, datastore) + + # Add processor capabilities from module + capabilities['supports_visual_selector'] = getattr(parent_module, 'supports_visual_selector', False) + capabilities['supports_text_filters_and_triggers'] = getattr(parent_module, 'supports_text_filters_and_triggers', False) + capabilities['supports_request_type'] = getattr(parent_module, 'supports_request_type', False) + app_rss_token = datastore.data['settings']['application'].get('rss_access_token'), c = [f"processor-{watch.get('processor')}"] diff --git a/changedetectionio/blueprint/ui/templates/edit.html b/changedetectionio/blueprint/ui/templates/edit.html index b6398638310..e76bea22920 100644 --- a/changedetectionio/blueprint/ui/templates/edit.html +++ b/changedetectionio/blueprint/ui/templates/edit.html @@ -45,14 +45,19 @@
+ {% if capabilities.supports_request_type %}
{{ render_field(form.fetch_backend, class="fetch-backend") }} @@ -203,6 +209,7 @@
+ {% endif %}
{% if capabilities.supports_browser_steps %} @@ -283,8 +290,7 @@

{{ _('Click here to Start') }}

- {% if watch['processor'] == 'text_json_diff' or watch['processor'] == 'image_ssim_diff' %} - + {% if capabilities.supports_text_filters_and_triggers %}
@@ -69,7 +73,9 @@
- {{ _('Create a shareable link') }} {{ _("Tip: You can also add 'shared' watches.") }} {{ _('More info') }} + + Tip: {{ tips | random | safe }} +