Backport changes from Pro (#1937)

Resolves tiny-pilot/tinypilot-pro#1516

This PR backports some of the changes from our Pro repo in order to keep
the diff between projects as small as possible to reduce the maintenance
admin.

Notes:

1. These changes were created by temporarily replacing the repo's `.git`
directory with that of TinyPilot Pro's `.git` directory and manually
reverting changes that seem relevant.
1. Seeing as we don't officially support Community running on a
cm4-based device, I decided to exclude any config specific to the cm4.
This partially affects the boot config and included EDID files.
1. I've tested these changes on a Voyager 2a device and it seems to be
working fine. <s>Also, I ran it on a Hobbyist pi4 device with usb
dongle, however I kept getting no video signal with this `ustreamer`
error in the logs:
> ERROR [194.294 ] -- No access to capture device: No such file or
directory
    
I wasn't able to figure out the issue yet, but I suspect my pi4 device
is faulty. I would appreciate a 2nd opinion from a reviewer.</s>
Nevermind, I just disabled the `tc358743` chip on the Voyager 2a to
mimic a Hobbyist device and it works.
1. These changes should create an empty diff when merging the
automatically generated PR back into the TinyPilot Pro repo.

<a data-ca-tag
href="https://codeapprove.com/pr/tiny-pilot/tinypilot/1937"><img
src="https://codeapprove.com/external/github-tag-allbg.png" alt="Review
on CodeApprove" /></a>

Author: jdeanwallace

.circleci/config.yml

@@ -117,34 +117,30 @@ jobs:
           name: Check code style
           command: ./dev-scripts/check-style
   decode_edid:
-    executor: python
+    executor: ubuntu
     resource_class: small
     steps:
       - checkout
       - run:
           name: Install edid-decode
           command: sudo apt-get update && sudo apt-get install --yes edid-decode
       - run:
-          name: Create virtual environment
-          command: python3 -m venv venv
-      - run:
-          name: Install requirements and decode the EDID
-          command: |
-            . venv/bin/activate
-            pip install --requirement requirements.txt
-            ./dev-scripts/decode-edid
+          name: Check pi4 EDID
+          command: edid-decode --check debian-pkg/usr/share/tinypilot/edid_pi4.hex
   check_python:
     executor: python
     steps:
       - checkout
       - run:
-          name: Create virtual environment
-          command: python3 -m venv venv
-      - run:
-          name: Install requirements and run build script
+          name: Install Python requirements of main app
           command: |
+            python3 -m venv venv
             . venv/bin/activate
             pip install --requirement dev_requirements.txt
+      - run:
+          name: Run check script
+          command: |
+            . venv/bin/activate
             ./dev-scripts/check-python
   check_javascript:
     executor: node

app/db/migrations/010-serial-terminal-create.sql

@@ -0,0 +1,11 @@
+-- Create a table for storing serial terminal settings.
+
+CREATE TABLE IF NOT EXISTS serial_terminal(
+    id INTEGER PRIMARY KEY,
+    port TEXT NOT NULL UNIQUE,
+    baud_rate INTEGER NOT NULL,
+    data_bits INTEGER NOT NULL,
+    stop_bits INTEGER NOT NULL,
+    parity TEXT NOT NULL,
+    flow_control TEXT NOT NULL
+);

app/static/js/app.js

@@ -368,6 +368,9 @@ menuBar.addEventListener("manage-users-dialog-requested", () => {
 menuBar.addEventListener("change-password-dialog-requested", () => {
   document.getElementById("change-password-overlay").show();
 });
+menuBar.addEventListener("serial-terminal-dialog-requested", () => {
+  document.getElementById("feature-pro-overlay").show();
+});
 menuBar.addEventListener("ctrl-alt-del-requested", () => {
   // Even though only the final keystroke matters, send them one at a time to
   // better match real user behavior. This ensures that the keystroke history

app/templates/custom-elements/menu-bar.html

@@ -323,6 +323,13 @@
             >Wake on LAN</a
           >
         </li>
+        <li class="item pro-badge" role="presentation">
+          <a
+            data-onclick-event="serial-terminal-dialog-requested"
+            role="menuitem"
+            >Serial Console</a
+          >
+        </li>
         <li class="item subgroup" role="presentation">
           <a role="menuitem">Keyboard Shortcuts</a>
           <ul class="items" role="group">

app/update/settings.py

@@ -33,41 +33,6 @@
 
 _SETTINGS_FILE_PATH = env.abs_path_in_home_dir('settings.yml')
 
-# To create a new EDID:
-#  1. Convert the existing EDID to binary using "edid2bin".
-#  2. Edit the binary using "AW EDID Editor v.02.00.13".
-#  3. Save the new EDID in binary format.
-#  4. Convert the binary EDID to a hex EDID using "make-edid".
-#    - Use the "--yaml" option if required.
-
-# Note: You may need to perform a few extra steps in order for the EDID
-# to conform to edid-decode's check - but only if the "Display Range Limits"
-# block changes. This is due to a bug in AW EDID Editor v.02.00.13 that doesn't
-# set the correct bytes.
-# To work around this:
-#  1. Open the new EDID in "AW EDID Editor v.3.0.10".
-#  2. Save the EDID to set correct bytes in the "Display Range Limits" block
-#  3. Re-open the EDID in "AW EDID Editor v.02.00.13" and re-set
-#     the screen size dimensions to 0 (both vertical and horizontal)
-_DEFAULT_TC358743_EDID = """
-00ffffffffffff005262769800888888
-2d1e0103800000781aee91a3544c9926
-0f50547fef8081c08140810081809500
-a9c081406140271f80f07138164038c0
-350000000000001eec2c80a070381a40
-3020350000000000001e000000fc0054
-696e7950696c6f740a202020000000fd
-00185a125010000a20202020202001aa
-02031ef14b010204131f2021223c3d3e
-2309070766030c00300080e2007f0000
-00000000000000000000000000000000
-00000000000000000000000000000000
-00000000000000000000000000000000
-00000000000000000000000000000000
-00000000000000000000000000000000
-0000000000000000000000000000008e
-""".strip()
-
 # Define default values for user-configurable TinyPilot settings. The YAML data
 # in _SETTINGS_FILE_PATH take precedence over these defaults.
 _DEFAULTS = {
@@ -77,7 +42,7 @@
     # settings file if they want to provide their own TLS keys.
     'tinypilot_manage_tls_keys': True,
     'ustreamer_desired_fps': video_service.DEFAULT_MJPEG_FRAME_RATE,
-    'ustreamer_edid': _DEFAULT_TC358743_EDID,
+    'ustreamer_edid': video_service.DEFAULT_EDID,
     'ustreamer_quality': video_service.DEFAULT_MJPEG_QUALITY,
     'ustreamer_h264_bitrate': video_service.DEFAULT_H264_BITRATE,
     'janus_stun_server': video_service.DEFAULT_H264_STUN_SERVER,

app/version.py

@@ -82,13 +82,18 @@ def latest_version():
         VersionRequestError: If an error occurred while making an HTTP request
             to the Gatekeeper API.
     """
+    # The URL is trusted because it's not based on user input.
+    url = f'{env.GATEKEEPER_BASE_URL}/community/available-update'
     try:
-        # The URL is trusted because it's not based on user input.
         # pylint: disable=line-too-long
         with urllib.request.urlopen(  # noqa: S310 # nosemgrep: dynamic-urllib-use-detected
-                f'{env.GATEKEEPER_BASE_URL}/community/available-update',
+                url,
                 timeout=10) as response:
             response_bytes = response.read()
+    except urllib.error.HTTPError as e:
+        message = e.fp.read().decode('utf-8').strip()
+        raise VersionRequestError(
+            f'Failed to request latest available version: {message}') from e
     except urllib.error.URLError as e:
         if (isinstance(e.reason, ssl.SSLCertVerificationError) and
                 e.reason.verify_message == 'certificate is not yet valid'):

app/version_test.py

@@ -1,3 +1,4 @@
+import io
 import json
 import tempfile
 import urllib.error
@@ -8,7 +9,7 @@
 import version
 
 
-class VersionTest(TestCase):
+class LocalVersionTest(TestCase):
 
     def setUp(self):
         # Run all unit tests with debug mode disabled.
@@ -63,6 +64,17 @@ def test_local_version_raises_file_error_when_file_is_not_utf_8(self):
                 with self.assertRaises(version.VersionFileError):
                     version.local_version()
 
+
+class LatestVersionTest(TestCase):
+
+    def setUp(self):
+        # Run all unit tests with version's debug mode enabled.
+        version_is_debug_patch = mock.patch.object(version,
+                                                   '_is_debug',
+                                                   return_value=True)
+        self.addCleanup(version_is_debug_patch.stop)
+        version_is_debug_patch.start()
+
     @mock.patch.object(urllib.request, 'urlopen')
     def test_latest_version_when_request_is_successful(self, mock_urlopen):
         mock_response = mock.Mock()
@@ -123,14 +135,20 @@ def test_latest_version_raises_request_error_when_response_missing_field(
     @mock.patch.object(urllib.request, 'urlopen')
     def test_latest_version_raises_request_error_when_request_fails(
             self, mock_urlopen):
-        mock_urlopen.side_effect = urllib.error.URLError(
-            'dummy error from gatekeeper', None)
+        urlopen_mock = mock.Mock(side_effect=urllib.error.HTTPError(
+            '127.0.0.1', 400, '400 Bad Request', None, io.BytesIO(
+                b'bad request')))
+        mock_urlopen.return_value.__enter__ = urlopen_mock
 
-        with self.assertRaises(version.VersionRequestError):
+        with self.assertRaises(version.VersionRequestError) as ctx:
             version.latest_version()
 
+        self.assertEqual(
+            'Failed to request latest available version: bad request',
+            str(ctx.exception))
+
 
-class DebugModeVersionTest(TestCase):
+class DebugModeLocalVersionTest(TestCase):
 
     def test_local_version_returns_dummy_version_when_in_debug_mode(self):
         # Enable debug mode.

app/video_service.py

@@ -1,16 +1,45 @@
+import contextlib
 import logging
 import subprocess
 
 import db.settings
 
+logger = logging.getLogger(__name__)
+
+# To create a new EDID:
+#  1. Convert the existing EDID to binary using "edid2bin".
+#  2. Edit the binary using "AW EDID Editor v.02.00.13".
+#  3. Save the new EDID in binary format.
+#  4. Convert the binary EDID to a hex EDID using "make-edid".
+#    - Use the "--yaml" option if required.
+#
+# Note: You may need to perform a few extra steps in order for the EDID
+# to conform to edid-decode's check - but only if the "Display Range Limits"
+# block changes. This is due to a bug in AW EDID Editor v.02.00.13 that doesn't
+# set the correct bytes.
+# To work around this:
+#  1. Open the new EDID in "AW EDID Editor v.3.0.10".
+#  2. Save the EDID to set correct bytes in the "Display Range Limits" block
+#  3. Re-open the EDID in "AW EDID Editor v.02.00.13" and re-set
+#     the screen size dimensions to 0 (both vertical and horizontal)
+_EDID_PI4_FILE = '/usr/share/tinypilot/edid_pi4.hex'
+
+
+def _get_default_edid():
+    with contextlib.suppress(FileNotFoundError):
+        with open(_EDID_PI4_FILE, encoding='utf-8') as file:
+            return file.read().strip()
+
+    return None
+
+
+DEFAULT_EDID = _get_default_edid()
 DEFAULT_MJPEG_FRAME_RATE = 30
 DEFAULT_MJPEG_QUALITY = 80
 DEFAULT_H264_BITRATE = 5000
 DEFAULT_H264_STUN_SERVER = None
 DEFAULT_H264_STUN_PORT = None
 
-logger = logging.getLogger(__name__)
-
 
 def restart():
     """Restarts the video streaming services for the remote screen.

bundler/bundle/install

@@ -25,8 +25,11 @@ if [[ -n "${FORCE_INSTALL+x}" ]]; then
 else
   # Restrict installation to only specific configurations.
 
-  # Install only on Raspberry Pi 4 Model B.
-  if ! grep --quiet "^Model\s*: Raspberry Pi 4 Model B" /proc/cpuinfo ; then
+  # Install only on supported hardware:
+  # - Raspberry Pi 4 Model B (e.g., Voyager 2)
+  if ! grep --quiet \
+    --regexp '^Model\s*: Raspberry Pi 4 Model B' \
+    /proc/cpuinfo; then
     echo 'You are trying to install on unsupported hardware.' >&2
     echo 'Visit https://github.com/tiny-pilot/tinypilot/ for more details.' >&2
     exit 1

debian-pkg/Dockerfile

@@ -4,12 +4,13 @@
 
 FROM debian:bullseye-20220328-slim AS build
 
+ENV DEBIAN_FRONTEND=noninteractive
 RUN set -exu && \
     apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y \
-      debhelper \
-      dh-virtualenv \
-      dpkg-dev
+    apt-get install --yes --no-install-recommends \
+      dpkg-dev \
+      devscripts \
+      equivs
 
 # Docker populates this value from the --platform argument. See
 # https://docs.docker.com/build/building/multi-platform/
@@ -73,16 +74,35 @@ Source: ${PKG_NAME}
 Section: net
 Priority: optional
 Maintainer: TinyPilot Support <support@tinypilotkvm.com>
-Build-Depends: debhelper (>= 11)
+Build-Depends:
+  debhelper (>= 11),
+  dh-virtualenv,
+  python3-dev,
+  python3-pip,
+  python3-venv
 
 Package: ${PKG_NAME}
 Architecture: ${PKG_ARCH}
-Depends: \${shlibs:Depends}, adduser, janus, nginx, python3, python3-pip, python3-venv, sudo, ustreamer
+Depends: \${shlibs:Depends},
+  \${misc:Depends},
+  adduser,
+  janus,
+  nginx,
+  python3,
+  sudo,
+  ustreamer
 Homepage: https://tinypilotkvm.com
 Description: Simple, easy-to-use KVM over IP
 XBS-Tinypilot-Version: ${TINYPILOT_VERSION}
 EOF
 
+# Install build dependencies based on Debian control file.
+RUN mk-build-deps \
+      --tool 'apt-get --option Debug::pkgProblemResolver=yes --no-install-recommends --yes -qq' \
+      --install \
+      --remove \
+      control
+
 RUN set -exu && \
     cat >changelog <<EOF
 tinypilot (${PKG_VERSION}) bullseye; urgency=medium