New configuration setting LINKBACKS_IGNORED_URLS_PATTERN

Author: Lucas-C

.pylintrc

@@ -1,5 +1,5 @@
 [MESSAGES CONTROL]
-disable = broad-except, missing-docstring, multiple-imports, too-few-public-methods, too-many-arguments, too-many-locals, too-many-positional-arguments
+disable = broad-except, missing-docstring, multiple-imports, too-few-public-methods, too-many-arguments, too-many-branches, too-many-locals, too-many-positional-arguments
 
 [FORMAT]
 max-line-length = 180

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Added
 
+- new configuration setting `LINKBACKS_IGNORED_URLS_PATTERN` to define some URLs that should never be considered for linkbacks (_e.g._ `youtube.com`)
 - manual execution mode: `python linkbacks.py $pelican_generated_html_file`
 
 ### Changed

pelican/plugins/linkbacks/linkbacks.py

@@ -9,6 +9,7 @@
 import json
 import logging
 import os
+import re
 import sys
 from os import makedirs
 from os.path import basename, splitext
@@ -32,6 +33,8 @@
 DEFAULT_USER_AGENT = 'pelican-plugin-linkbacks'
 DEFAULT_CERT_VERIFY = True
 DEFAULT_TIMEOUT = 3
+DEFAULT_IGNORED_URLS_PATTERN = 'deviantart.com|youtube.com'
+IMAGE_EXTENSIONS = ('.gif', '.jpg', '.pdf', '.png', '.svg')
 WEBMENTION_POSS_REL = ('webmention', 'http://webmention.org', 'http://webmention.org/', 'https://webmention.org', 'https://webmention.org/')
 
 LOGGER = logging.getLogger(__name__)
@@ -79,9 +82,12 @@ def process_all_links_of_an_article(config, cache, url, slug, content):
         if config.siteurl and link_url.startswith(config.siteurl):
             LOGGER.debug("Link url %s skipped because is starts with %s", link_url, config.siteurl)
             continue
-        if splitext(link_url)[1] in ('.gif', '.jpg', '.pdf', '.png', '.svg'):
+        if splitext(link_url)[1] in IMAGE_EXTENSIONS:
             LOGGER.debug("Link url %s skipped because it appears to be an image or PDF file", link_url)
             continue
+        if any(regex.search(link_url) for regex in config.ignored_urls_pattern):
+            LOGGER.debug("Link url %s skipped because it matches the ignored URLs pattern", link_url)
+            continue
         cache_status = cache.get_status(slug, link_url)
         if cache_status:
             LOGGER.debug("Link url %s skipped because it is present in cache with status: %s", link_url, cache_status)
@@ -128,6 +134,9 @@ def __init__(self, settings=None):
         self.cert_verify = settings.get('LINKBACKS_CERT_VERIFY', DEFAULT_CERT_VERIFY)
         self.timeout = settings.get('LINKBACKS_REQUEST_TIMEOUT', DEFAULT_TIMEOUT)
         self.user_agent = settings.get('LINKBACKS_USERAGENT', DEFAULT_USER_AGENT)
+        self.ignored_urls_pattern = settings.get('LINKBACKS_IGNORED_URLS_PATTERN', DEFAULT_IGNORED_URLS_PATTERN)
+        if self.ignored_urls_pattern and isinstance(self.ignored_urls_pattern, str):
+            self.ignored_urls_pattern = re.compile(self.ignored_urls_pattern)
 
 class Cache:
     def __init__(self, config, data):