Merge pull request #127 from Falldog/fix/issue-124-unicode-argv

Falldog · web-flow · commit 5c6b0c3f27b3 · 2025-08-17T08:51:55.000+08:00
fix: issue 124 unicode sys.argv
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,16 @@
 # Pyconcrete Changelog
 
+## 1.1.? (????)
+
+### Bug fixes
+* Add `__file__` for entry pye script https://github.com/Falldog/pyconcrete/issues/123
+* Fix `sys.argv` to support unicode https://github.com/Falldog/pyconcrete/issues/124
+  * Python 3.7 on Windows is not support
+
+### Breaking Changes
+* Default enable Python utf8 mode, ref doc: https://docs.python.org/3/library/os.html#utf8-mode
+
+
 ## 1.1.0 (2025-07-24)
 
 ### Features
diff --git a/src/pyconcrete_exe/pyconcrete_exe.c b/src/pyconcrete_exe/pyconcrete_exe.c
@@ -18,38 +18,52 @@
     #define MAGIC_OFFSET 8
 #endif
 
+// WIN32 platform use wmain, all string related functions should change to wchar_t version
+#ifdef WIN32
+    #define _CHAR                                       wchar_t
+    #define _T(s)                                       L##s
+    #define _fopen                                      _wfopen
+    #define _strncmp                                    wcsncmp
+    #define _strlen                                     wcslen
+    #define _PyConfig_SetArgv                           PyConfig_SetArgv
+    #define _PyConfig_SetString                         PyConfig_SetString
+    #define _PyUnicode_FromStringAndSize                PyUnicode_FromWideChar
+#else
+    #define _CHAR                                       char
+    #define _T(s)                                       s
+    #define _fopen                                      fopen
+    #define _strncmp                                    strncmp
+    #define _strlen                                     strlen
+    #define _PyConfig_SetArgv                           PyConfig_SetBytesArgv
+    #define _PyConfig_SetString                         PyConfig_SetBytesString
+    #define _PyUnicode_FromStringAndSize                PyUnicode_FromStringAndSize
+#endif
+
 
 int createAndInitPyconcreteModule();
-int execPycContent(PyObject* pyc_content, const char* filepath);
-int runFile(const char* filepath);
-PyObject* getFullPath(const char* filepath);
+int execPycContent(PyObject* pyc_content, const _CHAR* filepath);
+int runFile(const _CHAR* filepath);
+int prependSysPath0(const _CHAR* script_path);
+void initPython(int argc, _CHAR *argv[]);
+PyObject* getFullPath(const _CHAR* filepath);
 
 
+#ifdef WIN32
+int wmain(int argc, wchar_t *argv[])
+#else
 int main(int argc, char *argv[])
+#endif
 {
-#if PY_MAJOR_VERSION >= 3
-    int i, len;
     int ret = RET_OK;
-    wchar_t** argv_ex = NULL;
-    argv_ex = (wchar_t**) malloc(sizeof(wchar_t*) * argc);
-    for(i=0 ; i<argc ; ++i)
-    {
-        len = mbstowcs(NULL, argv[i], 0);
-        argv_ex[i] = (wchar_t*) malloc(sizeof(wchar_t) * (len+1));
-        mbstowcs(argv_ex[i], argv[i], len);
-        argv_ex[i][len] = 0;
-    }
-#else
-    char** argv_ex = argv;
-#endif
 
-    Py_SetProgramName(argv_ex[0]);  /* optional but recommended */
     // PyImport_AppendInittab must set up before Py_Initialize
     if (PyImport_AppendInittab("_pyconcrete", PyInit__pyconcrete) == -1)
     {
         fprintf(stderr, "Error, can't load embedded _pyconcrete correctly!\n");
         return RET_FAIL;
     }
+
+    initPython(argc, argv);
     Py_Initialize();
     PyGILState_Ensure();
 
@@ -62,13 +76,40 @@ int main(int argc, char *argv[])
 
     if(argc >= 2)
     {
-        if(argc == 2 && (strncmp(argv[1], "-v", 3)==0 || strncmp(argv[1], "--version", 10)==0))
+        if(argc == 2 && (_strncmp(argv[1], _T("-v"), 3)==0 || _strncmp(argv[1], _T("--version"), 10)==0))
         {
             printf("pyconcrete %s [Python %s]\n", TOSTRING(PYCONCRETE_VERSION), TOSTRING(PY_VERSION));  // defined by build-backend
         }
         else
         {
+#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION <=7
+    #if defined(WIN32)
+            PySys_SetArgv(argc-1, argv+1);
+    #else
+            int i, len;
+            wchar_t** argv_ex = NULL;
+            argv_ex = (wchar_t**) malloc(sizeof(wchar_t*) * argc);
+            // setup
+            for(i=0 ; i<argc ; ++i)
+            {
+                len = mbstowcs(NULL, argv[i], 0);
+                argv_ex[i] = (wchar_t*) malloc(sizeof(wchar_t) * (len+1));
+                mbstowcs(argv_ex[i], argv[i], len);
+                argv_ex[i][len] = 0;
+            }
+
+            // set argv
             PySys_SetArgv(argc-1, argv_ex+1);
+
+            // release
+            for(i=0 ; i<argc ; ++i)
+            {
+                free(argv_ex[i]);
+            }
+    #endif
+#else
+            prependSysPath0(argv[1]);
+#endif // PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION <=7
             ret = runFile(argv[1]);
         }
     }
@@ -91,17 +132,79 @@ int main(int argc, char *argv[])
     }
 
     Py_Finalize();
+    return ret;
+}
 
-#if PY_MAJOR_VERSION >= 3
-    for(i=0 ; i<argc ; ++i)
+
+void initPython(int argc, _CHAR *argv[]) {
+#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION <=7
+    #if defined(WIN32)
+        Py_SetProgramName(argv[0]);
+    #else
+        int len = mbstowcs(NULL, argv[0], 0);
+        wchar_t* arg0 = (wchar_t*) malloc(sizeof(wchar_t) * (len+1));
+        mbstowcs(arg0, argv[0], len);
+        arg0[len] = 0;
+        Py_SetProgramName(arg0);
+    #endif
+#else
+    PyStatus status;
+
+    // ----------
+    // PyPreConfig
+    // ----------
+    // On Windows platform invoke pyconcrete by subprocess may changed the console encoding to cp1252
+    // force to set utf8 mode to avoid the issue.
+    PyPreConfig preconfig;
+    PyPreConfig_InitPythonConfig(&preconfig);
+    preconfig.utf8_mode = 1;
+
+    status = Py_PreInitialize(&preconfig);
+    if (PyStatus_Exception(status)) {
+        goto INIT_EXCEPTION;
+    }
+
+    // ----------
+    // PyConfig
+    // ----------
+    PyConfig config;
+    PyConfig_InitPythonConfig(&config);
+    config.parse_argv = 0;
+    config.isolated = 1;
+
+    // Set program_name as pyconcrete. (Implicitly preinitialize Python)
+    status = _PyConfig_SetString(&config, &config.program_name, argv[0]);
+    if (PyStatus_Exception(status)) {
+        goto INIT_EXCEPTION;
+    }
+
+    // Decode command line arguments. (Implicitly preinitialize Python)
+    status = _PyConfig_SetArgv(&config, argc-1, argv+1);
+    if (PyStatus_Exception(status))
     {
-        free(argv_ex[i]);
+        goto INIT_EXCEPTION;
     }
-    free(argv_ex);
-#endif
-    return ret;
+
+    status = Py_InitializeFromConfig(&config);
+    if (PyStatus_Exception(status))
+    {
+        goto INIT_EXCEPTION;
+    }
+    PyConfig_Clear(&config);
+    return;
+
+INIT_EXCEPTION:
+    PyConfig_Clear(&config);
+    if (PyStatus_IsExit(status))
+    {
+        return status.exitcode;
+    }
+    // Display the error message and exit the process with non-zero exit code
+    Py_ExitStatusException(status);
+#endif // ifdef PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION <=7
 }
 
+
 int createAndInitPyconcreteModule()
 {
     int ret = 0;
@@ -137,7 +240,8 @@ int createAndInitPyconcreteModule()
     return ret;
 }
 
-int execPycContent(PyObject* pyc_content, const char* filepath)
+
+int execPycContent(PyObject* pyc_content, const _CHAR* filepath)
 {
     int ret = RET_OK;
     PyObject* py_marshal = NULL;
@@ -186,7 +290,8 @@ int execPycContent(PyObject* pyc_content, const char* filepath)
     return ret;
 }
 
-int runFile(const char* filepath)
+
+int runFile(const _CHAR* filepath)
 {
     FILE* src = NULL;
     char* content = NULL;
@@ -196,7 +301,7 @@ int runFile(const char* filepath)
     PyObject* py_plaint_content = NULL;
     PyObject* py_args = NULL;
 
-    src = fopen(filepath, "rb");
+    src = _fopen(filepath, _T("rb"));
     if(src == NULL)
     {
         return RET_FAIL;
@@ -230,17 +335,51 @@ int runFile(const char* filepath)
     return ret;
 }
 
-PyObject* getFullPath(const char* filepath)
+
+/*
+    PySys_SetArgv is deprecated since python 3.11. It's original behavior will insert script's directory into sys.path.
+    It's replace by PyConfig, but PyConfig only update sys.path when executing Py_Main or Py_RunMain.
+    So it's better to update sys.path by pyconcrete.
+ */
+int prependSysPath0(const _CHAR* script_path)
+{
+    // script_dir = os.path.dirname(script_path)
+    // sys.path.insert(0, script_dir)
+    int ret = RET_OK;
+
+    PyObject* py_script_path = getFullPath(script_path);
+    PyObject* path_module = PyImport_ImportModule("os.path");
+    PyObject* dirname_func = PyObject_GetAttrString(path_module, "dirname");
+    PyObject* args = Py_BuildValue("(O)", py_script_path);
+    PyObject* script_dir = PyObject_CallObject(dirname_func, args);
+
+    PyObject* sys_path = PySys_GetObject("path");
+    if (PyList_Insert(sys_path, 0, script_dir) < 0) {
+        ret = RET_FAIL;
+    }
+
+    Py_XDECREF(py_script_path);
+    Py_XDECREF(path_module);
+    Py_XDECREF(dirname_func);
+    Py_XDECREF(args);
+    Py_XDECREF(script_dir);
+    return ret;
+}
+
+
+PyObject* getFullPath(const _CHAR* filepath)
 {
     // import os.path
     // return os.path.abspath(filepath)
     PyObject* path_module = PyImport_ImportModule("os.path");
     PyObject* abspath_func = PyObject_GetAttrString(path_module, "abspath");
-    PyObject* args = Py_BuildValue("(s)", filepath);
-    PyObject* obj = PyObject_CallObject(abspath_func, args);
+    PyObject* py_filepath = _PyUnicode_FromStringAndSize(filepath, _strlen(filepath));
+    PyObject* args = Py_BuildValue("(O)", py_filepath);
+    PyObject* py_file_abspath = PyObject_CallObject(abspath_func, args);
 
     Py_XDECREF(path_module);
     Py_XDECREF(abspath_func);
+    Py_XDECREF(py_filepath);
     Py_XDECREF(args);
-    return obj;
+    return py_file_abspath;
 }
diff --git a/tests/test_exe.py b/tests/test_exe.py
@@ -12,7 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import os.path
+import platform
 import subprocess
+import sys
+
+import pytest
 
 
 def test_exe__execute_an_non_exist_file(venv_exe):
@@ -85,6 +89,28 @@ def test_exe__sys_argv__more_arguments(venv_exe, pye_cli, tmpdir):
     assert output == f'{pye_path} -a -b -c'
 
 
+@pytest.mark.skipif(platform.system() == 'Windows' and sys.version_info < (3, 8), reason="Windows requires python3.8")
+def test_exe__sys_argv__in_unicode(venv_exe, pye_cli, tmpdir):
+    # prepare
+    pye_path = (
+        pye_cli.setup(tmpdir, 'test_sys_argv')
+        .source_code(
+            """
+import sys
+print(" ".join(sys.argv))
+            """.strip()
+        )
+        .get_encrypt_path()
+    )
+
+    # execution
+    output = venv_exe.pyconcrete(pye_path, '早安', '=', 'おはようございます')
+    output = output.strip()
+
+    # verification
+    assert output == f'{pye_path} 早安 = おはようございます'
+
+
 def test_exe__import_pyconcrete__venv_exe__validate__file__(venv_exe, pye_cli, tmpdir):
     """
     compare to test_lib__import_pyconcrete__venv_lib__validate__file__
