Sweep the tree for everything conditional on python 2.

The only thing explicitly left out is the walk relted code in fileio
which is rather weedy and being tackled in a separate change.

Author: albu-diku

mig/server/grid_events.py

@@ -538,7 +538,7 @@ def __update_rule_monitor(
                     # Fire 'modified' events for all dirs and files in subpath
                     # to ensure that all rule files are loaded
 
-                    for ent in scandir(src_path):
+                    for ent in os.scandir(src_path):
                         if ent.is_dir(follow_symlinks=True):
 
                             # logger.debug('(%s) Dispatch DirCreatedEvent for: %s'
@@ -1139,7 +1139,7 @@ def __update_file_monitor(self, event):
                     # For create this occurs by eg. mkdir -p 'path/subpath/subpath2'
                     # or 'cp -rf'
 
-                    for ent in scandir(src_path):
+                    for ent in os.scandir(src_path):
                         if ent.is_dir(follow_symlinks=True):
                             vgrid_sub_path = strip_base_dirs(ent.path)
 
@@ -1491,7 +1491,7 @@ def add_vgrid_file_monitor(configuration, vgrid_name, path):
 
                 # Traverse dirs for subdirs created since last run
 
-                for ent in scandir(vgrid_files_path):
+                for ent in os.scandir(vgrid_files_path):
                     if ent.is_dir(follow_symlinks=True):
                         vgrid_sub_path = strip_base_dirs(ent.path)
                         # Force utf8 everywhere to avoid encoding issues
@@ -1839,7 +1839,7 @@ def monitor(configuration, vgrid_name):
                                          configuration.vgrid_triggers)
                 all_trigger_rules.append(rule_path)
     else:
-        for ent in scandir(vgrid_home):
+        for ent in os.scandir(vgrid_home):
             if configuration.vgrid_triggers in ent.name:
                 rule_path = ent.path
                 all_trigger_rules.append(rule_path)
@@ -1981,7 +1981,7 @@ def monitor(configuration, vgrid_name):
 
     # Each top vgrid gets is own process
 
-    for ent in scandir(configuration.vgrid_home):
+    for ent in os.scandir(configuration.vgrid_home):
         vgrid_files_path = os.path.join(configuration.vgrid_files_home,
                                         ent.name)
         if os.path.isdir(ent.path) and os.path.isdir(vgrid_files_path):

mig/shared/base.py

@@ -736,23 +736,13 @@ def native_args(argv):
     https://docs.python.org/3/library/sys.html#sys.argv
     for further details.
     """
-    if sys.version_info[0] >= 3:
-        return [os.fsencode(arg) for arg in argv]
-    else:
-        return argv
+    return [os.fsencode(arg) for arg in argv]
 
 
 def NativeStringIO(initial_value=''):
-    """Mock StringIO pseudo-class to create a StringIO matching the native
-    string coding form. That is a BytesIO with utf8 on python 2 and unicode
-    StringIO otherwise. Optional string helpers are automatically converted
-    accordingly.
+    """Pseudo-class wrapper to return a StringIO. This is a unicode StringIO.
     """
-    if sys.version_info[0] >= 3:
-        return io.StringIO(initial_value)
-    else:
-        from StringIO import StringIO
-        return StringIO(initial_value)
+    return io.StringIO(initial_value)
 
 
 def DefaultStringIO(initial_value=''):

mig/shared/cgiscriptstub.py

@@ -117,9 +117,8 @@ def finish_cgi_script(configuration, backend, output_format, ret_code, ret_msg,
         #logger.debug("flush stdout")
         sys.stdout.flush()
         #logger.debug("write content: %s" % [output[:64], '..', output[-64:]])
-        # NOTE: always output native strings to stdout but use raw buffer
-        #       for byte output on py3 as explained above.
-        if sys.version_info[0] < 3 or is_default_str_coding(output):
+        # NOTE: use raw buffer for byte output as explained above
+        if is_default_str_coding(output):
             sys.stdout.write(output)
         else:
             sys.stdout.buffer.write(output)

mig/shared/compat.py

@@ -36,29 +36,10 @@
 import codecs
 import io
 import sys
-# NOTE: StringIO is only available in python2
-try:
-    import StringIO
-except ImportError:
-    StringIO = None
 
-PY2 = sys.version_info[0] < 3
 _TYPE_UNICODE = type(u"")
 
 
-if PY2:
-    class SimpleNamespace(dict):
-        """Bare minimum SimpleNamespace for Python 2."""
-
-        def __getattribute__(self, name):
-            if name == '__dict__':
-                return dict(**self)
-
-            return self[name]
-else:
-    from types import SimpleNamespace
-
-
 def _is_unicode(val):
     """Return boolean indicating if the value is a unicode string.
 
@@ -72,34 +53,9 @@ def ensure_native_string(string_or_bytes):
     """Given a supplied input which can be either a string or bytes
     return a representation providing string operations while ensuring that
     its contents represent a valid series of textual characters.
-
-    Arrange identical operation across python 2 and 3 - specifically,
-    the presence of invalid UTF-8 bytes (thus the input not being a
-    valid textual string) will trigger a UnicodeDecodeError on PY3.
-    Force the same to occur on PY2.
     """
-    if PY2:
-        # Simulate decoding done by PY3 to trigger identical exceptions
-        # note the use of a forced "utf8" encoding value: this function
-        # is generally used to wrap, for example, substitutions of values
-        # into strings that are defined in the source code. In Python 3
-        # these are mandated to be UTF-8, and thus decoding as "utf8" is
-        # what will be attempted on supplied input. Match it.
-        textual_output = codecs.encode(string_or_bytes, 'utf8')
-    elif not _is_unicode(string_or_bytes):
+    if not _is_unicode(string_or_bytes):
         textual_output = str(string_or_bytes, 'utf8')
     else:
         textual_output = string_or_bytes
     return textual_output
-
-
-def NativeStringIO(initial_value=''):
-    """Mock StringIO pseudo-class to create a StringIO matching the native
-    string coding form. That is a BytesIO with utf8 on python 2 and unicode
-    StringIO otherwise. Optional string helpers are automatically converted
-    accordingly.
-    """
-    if PY2 and StringIO is not None:
-        return StringIO.StringIO(initial_value)
-    else:
-        return io.StringIO(initial_value)

mig/shared/configuration.py

@@ -34,6 +34,7 @@
 from builtins import object
 
 import base64
+from configparser import ConfigParser
 import copy
 import datetime
 import functools
@@ -50,13 +51,6 @@
 standard_library.install_aliases()
 
 
-# NOTE: should be handled by future aliases but fails in PAM C extension
-if sys.version_info[0] < 3:
-    # NOTE: always use native py2 version here - new one causes unicode mess
-    from ConfigParser import ConfigParser
-else:
-    from configparser import ConfigParser
-
 # NOTE: protect migrid import from autopep8 reordering
 try:
     from mig.shared.base import force_native_str

mig/shared/defaults.py

@@ -37,17 +37,8 @@
 MIG_BASE = os.path.realpath(os.path.join(os.path.dirname(__file__), '../..'))
 MIG_ENV = os.getenv('MIG_ENV', 'default')
 
-# NOTE: python3 switched strings to use unicode by default in contrast to bytes
-#       in python2. File systems remain with utf8 however so we need to
-#       carefully handle a lot of cases of either encoding to utf8 or decoding
-#       to unicode depending on the python used.
-#       Please refer to the helpers in shared.base for actual handling of it.
-if sys.version_info[0] >= 3:
-    default_str_coding = 'unicode'
-    default_fs_coding = 'utf8'
-else:
-    default_str_coding = 'utf8'
-    default_fs_coding = 'utf8'
+default_str_coding = 'unicode'
+default_fs_coding = 'utf8'
 
 CODING_KINDS = (STR_KIND, FS_KIND) = ('__STR__', '__FS__')
 

mig/shared/output.py

@@ -651,7 +651,7 @@ def txt_format(configuration, ret_val, ret_msg, out_obj):
         lines = [status_line] + lines
 
     # NOTE: careful handling required for binary on python3+
-    if sys.version_info[0] > 2 and binary_output:
+    if binary_output:
         return b''.join(lines)
     else:
         return ''.join(lines)
@@ -2646,7 +2646,7 @@ def html_format(configuration, ret_val, ret_msg, out_obj):
                                          user_widgets))
 
     # NOTE: careful handling required for binary on python3+
-    if sys.version_info[0] > 2 and binary_output:
+    if binary_output:
         return b''.join(lines)
     else:
         return '\n'.join(lines)

mig/shared/ssh.py

@@ -44,9 +44,8 @@
     # Paramiko not available - imported fom griddaemons so fail gracefully
     paramiko = None
 
-from mig.shared.base import client_id_dir, force_utf8
+from mig.shared.base import client_id_dir, force_utf8, NativeStringIO
 from mig.shared.conf import get_resource_exe, get_configuration_object
-from mig.shared.compat import NativeStringIO
 from mig.shared.defaults import ssh_conf_dir
 from mig.shared.safeeval import subprocess_popen, subprocess_pipe
 

mig/shared/url.py

@@ -42,20 +42,9 @@
 import base64
 import os
 import sys
-
-# NOTE: moved to urllib.parse in python3 and are re-exposed with future.
-#       Other modules should import helpers from here for consistency.
-# TODO: handle the unicode returned by python3 and future versions!
-#       Perhaps switch to suggested "easiest option" from
-#       http://python-future.org/compatible_idioms.html#urllib-module
-#       once we have unicode/bytecode mix-up sorted out.
-if sys.version_info[0] >= 3:
-    from urllib.parse import quote, unquote, urlencode, parse_qs, parse_qsl, \
-        urlsplit, urlparse, urljoin
-    from urllib.request import urlopen
-else:
-    from urllib import quote, unquote, urlencode, urlopen
-    from urlparse import parse_qs, parse_qsl, urlsplit, urlparse, urljoin
+from urllib.parse import quote, unquote, urlencode, parse_qs, parse_qsl, \
+    urlsplit, urlparse, urljoin
+from urllib.request import urlopen
 
 try:
     from mig.shared.base import force_utf8, force_native_str

mig/unittest/testcore.py

@@ -38,7 +38,7 @@
 
 sys.path.append(os.path.realpath(
     os.path.join(os.path.dirname(__file__), "../..")))
-from tests.support import MIG_BASE, PY2, is_path_within
+from tests.support import MIG_BASE, is_path_within
 
 from mig.shared.base import client_id_dir, client_dir_id, get_short_id, \
     invisible_path, allow_script, brief_list