Fix Unicode handling and add test for Japanese Kanji characters

Author: jaccz

.gitignore

@@ -20,4 +20,5 @@
 pict
 *.o
 .DS_Store
-build
+build
+.vscode/settings.json

cli/ctokenizer.cpp

@@ -574,27 +574,27 @@ wstring ConstraintsTokenizer::getParameterName()
 //
 double ConstraintsTokenizer::getNumber()
 {
-    // declare new stream from text we'd like to parse
-    //  then try to get numeric value preserving old and new
-    //  position within a stream to properly update cursor
     wstring substring( _currentPosition, _constraintsText.end() );
-    wistringstream ist( substring );
-
-    unsigned int positionBefore = (unsigned int) ist.tellg();
-
+    
+    size_t charsConsumed = 0;
     double number;
-    ist>>number;
-
-    if (ist.rdstate() & ios::failbit)
+    
+    try
+    {
+        number = stod( substring, &charsConsumed );
+    }
+    catch( ... )
     {
         throw CSyntaxError( SyntaxErrorType::NotNumericValue, _currentPosition );
     }
-
-    // success, update current cursor position
-    unsigned int difference  =  (unsigned int) ist.tellg() - positionBefore;
-    _currentPosition += difference;
 
-    return ( number );
+    if( charsConsumed == 0 )
+    {
+        throw CSyntaxError( SyntaxErrorType::NotNumericValue, _currentPosition );
+    }
+
+    _currentPosition += charsConsumed;
+    return( number );
 }
 
 //

cli/mparser.cpp

@@ -1,5 +1,6 @@
 #include <fstream>
 #include <sstream>
+#include <locale>
 #include "model.h"
 using namespace std;
 
@@ -442,6 +443,8 @@ bool CModelData::readModel( const wstring& filePath )
         PrintMessage( InputDataError, L"Couldn't open file:", filePath.data() );
         return( false );
     }
+    // Use the current global locale (set to UTF-8 in main) for decoding
+    file.imbue( std::locale() );
 
     wstring line;
 
@@ -543,6 +546,8 @@ bool CModelData::ReadRowSeedFile( const wstring& filePath )
         PrintMessage( InputDataError, L"Couldn't open file:", filePath.data() );
         return( false );
     }
+    // Use the current global locale (set to UTF-8 in main) for decoding
+    file.imbue( std::locale() );
     wstring line;
 
     // parameter names

cli/pict.cpp

@@ -5,12 +5,14 @@
 
 #include <ctime>
 #include <cstring>
+#include <iostream>
 #include <locale>
 #include <stdexcept>
 using namespace std;
 
 #include "cmdline.h"
 #include "gcd.h"
+#include "strings.h"
 using namespace pictcli_gcd;
 
 //
@@ -119,9 +121,24 @@ int __cdecl wmain
     IN wchar_t* args[]
     )
 {
+    // Align all wide I/O with the current environment locale (UTF-8 on modern systems)
+    try
+    {
+        std::locale loc( "" );
+        std::locale::global( loc );
+        std::wcout.imbue( loc );
+        std::wcerr.imbue( loc );
+    }
+    catch( const std::runtime_error& )
+    {
+        // Fall back to the classic locale if the environment locale is unavailable
+        std::locale::global( std::locale::classic() );
+    }
+
     wstring output;
     int ret = execute( argc, args, output );
-    wcout << output;
+
+    std::wcout << output;
 
     return ret;
 }

test/bug/.tests

@@ -37,6 +37,7 @@ bug020.txt /d:*     -> SUCCESS
 bug021.txt          -> SUCCESS
 bug022.txt          -> SUCCESS
 bug023.txt /o:1     -> SUCCESS
+bug024.txt          -> SUCCESS
 
 bug002.txt          -> BAD_MODEL
 bug003.txt          -> BAD_MODEL

test/bug/bug024.txt

@@ -0,0 +1,7 @@
+# Testing Japanese Kanji characters in parameter names and constraints
+予:Hoge,X,Y,Z
+予定日時:Scheduled,NotScheduled
+Foo:Bar,Baz,A,B
+
+if [予定日時] = "Scheduled" then [Foo] = "Bar";
+if [予] = "Hoge" then [Foo] <> "Bar";