TInputLine: add Unicode support

This is great news!

Author: magiblot

README.md

@@ -255,8 +255,6 @@ KeyDownEvent {
 ```
 So, in short: views designed without Unicode input in mind will continue to work exactly as they did before, and views which want to be Unicode-aware will have no issues in being so.
 
-For the time being, none of the builtin views have been modified to support Unicode input.
-
 ## Displaying Unicode text
 
 The original design of Turbo Vision uses 16 bits to represent a *screen cell*—8 bit for a character and 8 bit for [BIOS color attributes](https://en.wikipedia.org/wiki/BIOS_color_attributes).
@@ -380,7 +378,7 @@ Here is what it looks like:
 
 Support for creating Unicode-aware views is in place, but most of the views that are part of the original Turbo Vision library have not been adapted to handle Unicode.
 
-* At least `TFrame`, `THistoryViewer`, `TListViewer`, `TInputLine` and `TMenuBox` are able to display Unicode text properly.
+* At least `TFrame`, `THistoryViewer`, `TListViewer` and `TMenuBox` are able to display Unicode text properly.
 * Automatic shortcuts in `TMenuBox` won't work with Unicode text, as shortcuts are still compared against `event.keyDown.charScan.charCode`.
-* `TInputLine` can display but not process Unicode text, as it still believes that every byte is one column wide. This can lead to inconsistencies, because some non-ASCII characters can be represented in the active codepage. For instance, if using CP437, it is possible to type `ç` into a `TInputLine`, but it will be a narrow character instead of a UTF-8 sequence. If you manage to place UTF-8 text in a `TInputLine` (e.g. by selecting a file in a `TFileDialog`), the caret will appear to be out of sync as its position is measured assuming a single-byte encoding.
+* `TInputLine` can display and process Unicode text.
 * `TEditor` assumes a single-byte encoding both when handling input events and when displaying text. So it won't display UTF-8 but at least it has a consistent behaviour.

include/tvision/dialogs.h

@@ -185,7 +185,9 @@ class TInputLine : public TView
     Boolean canScroll( int delta );
     int mouseDelta( TEvent& event );
     int mousePos( TEvent& event );
+    int displayedPos( int pos );
     void deleteSelect();
+    void deleteCurrent();
     void adjustSelectBlock();
     void saveState();
     void restoreState();

include/tvision/ttext.h

@@ -114,6 +114,10 @@ class TText {
 
 public:
 
+    static size_t next(TStringView text);
+    static size_t prev(TStringView text, size_t index);
+    static size_t wseek(TStringView text, int count);
+
 #ifndef __BORLANDC__
     static void eat(TScreenCell *cell, size_t n, size_t &width, std::string_view src, size_t &bytes);
     static void next(std::string_view src, size_t &bytes, size_t &width);
@@ -122,6 +126,61 @@ class TText {
 
 };
 
+#ifdef __BORLANDC__
+
+inline size_t TText::next(TStringView text)
+{
+    return text.size() ? 1 : 0;
+}
+
+inline size_t TText::prev(TStringView text, size_t index)
+{
+    return index ? 1 : 0;
+}
+
+inline size_t TText::wseek(TStringView text, int count)
+{
+    return count > 0 ? count : 0;
+}
+
+#else
+
+inline size_t TText::next(TStringView text)
+{
+    if (text.size()) {
+        std::mbstate_t state {};
+        int64_t len = std::mbrtowc(nullptr, text.data(), text.size(), &state);
+        return len <= 1 ? 1 : len;
+    }
+    return 0;
+}
+
+inline size_t TText::prev(TStringView text, size_t index)
+{
+    if (index) {
+        // Try reading backwards character by character, until a valid
+        // character is found. This tolerates invalid characters.
+        size_t lead = std::min<size_t>(index, 4);
+        for (size_t i = 1; i <= lead; ++i) {
+            std::mbstate_t state {};
+            int64_t size = std::mbrtowc(nullptr, &text[index - i], i, &state);
+            if (size > 0)
+                return size == i ? i : 1;
+        }
+        return 1;
+    }
+    return 0;
+}
+
+inline size_t TText::wseek(TStringView text, int count)
+{
+    size_t index = 0, remainder = 0;
+    wseek(text, index, remainder, count);
+    return index;
+}
+
+#endif
+
 #ifndef __BORLANDC__
 
 inline void TText::eat( TScreenCell *cell, size_t n, size_t &width,

source/tvision/tinputli.cpp

@@ -111,15 +111,15 @@ void TInputLine::draw()
         {
         if( canScroll(-1) )
             b.moveChar( 0, leftArrow, getColor(4), 1 );
-        l = selStart - firstPos;
-        r = selEnd - firstPos;
+        l = displayedPos(selStart) - firstPos;
+        r = displayedPos(selEnd) - firstPos;
         l = max( 0, l );
         r = min( size.x - 2, r );
         if (l <  r)
             b.moveChar( l+1, 0, getColor(3), r - l );
         }
     writeLine( 0, 0, size.x, size.y, b );
-    setCursor( curPos-firstPos+1, 0);
+    setCursor( displayedPos(curPos)-firstPos+1, 0);
 }
 
 void TInputLine::getData( void *rec )
@@ -153,8 +153,14 @@ int TInputLine::mousePos( TEvent& event )
     mouse.x = max( mouse.x, 1 );
     int pos = mouse.x + firstPos - 1;
     pos = max( pos, 0 );
-    pos = min( pos, strlen(data) );
-    return pos;
+    TStringView text = data;
+    pos = min( pos, strwidth(text) );
+    return TText::wseek(text, pos);
+}
+
+int TInputLine::displayedPos( int pos )
+{
+    return strwidth( TStringView(data, pos) );
 }
 
 void  TInputLine::deleteSelect()
@@ -166,6 +172,17 @@ void  TInputLine::deleteSelect()
         }
 }
 
+void TInputLine::deleteCurrent()
+{
+    TStringView text = data;
+    if( curPos < text.size() )
+        {
+        selStart = curPos;
+        selEnd = curPos + TText::next(text.substr(curPos));
+        deleteSelect();
+        }
+}
+
 void TInputLine::adjustSelectBlock()
 {
     if (curPos < anchor)
@@ -243,7 +260,7 @@ void TInputLine::handleEvent( TEvent& event )
     static char padKeys[] = {0x47,0x4b,0x4d,0x4f,0x73,0x74, 0};
     TView::handleEvent(event);
 
-    int delta, i;
+    int delta, i, len, curWidth;
     if( (state & sfSelected) != 0 )
         switch( event.what )
             {
@@ -296,12 +313,10 @@ void TInputLine::handleEvent( TEvent& event )
                 switch( event.keyDown.keyCode )
                     {
                     case kbLeft:
-                        if( curPos > 0 )
-                            curPos--;
+                        curPos -= TText::prev(TStringView(data), curPos);
                         break;
                     case kbRight:
-                        if( curPos < strlen(data) )
-                            curPos++;
+                        curPos += TText::next(TStringView(data+curPos));
                         break;
                     case kbHome:
                         curPos =  0;
@@ -312,41 +327,39 @@ void TInputLine::handleEvent( TEvent& event )
                     case kbBack:
                         if( curPos > 0 )
                             {
-                            strcpy( data+curPos-1, data+curPos );
-                            curPos--;
+                            TStringView text = data;
+                            int len = TText::prev(text, curPos);
+                            memmove( data+curPos-len, data+curPos, text.size()-curPos+1 );
+                            curPos -= len;
                             checkValid(True);
                             }
                         break;
                     case kbDel:
                         if( selStart == selEnd )
-                            if( curPos < strlen(data) )
-                                {
-                                selStart = curPos;
-                                selEnd = curPos + 1;
-                                }
-                        deleteSelect();
+                            deleteCurrent();
+                        else
+                            deleteSelect();
                         checkValid(True);
                         break;
                     case kbIns:
                         setState(sfCursorIns, Boolean(!(state & sfCursorIns)));
                         break;
                     default:
-                        if( event.keyDown.charScan.charCode >= ' ' )
+                        if( (len = event.keyDown.textLength) )
                             {
                             deleteSelect();
                             if( (state & sfCursorIns) != 0 )
-                                /* The following must be a signed comparison! */
-                                if( curPos < (int) strlen(data) )
-                                    strcpy( data + curPos, data + curPos + 1 );
+                                deleteCurrent();
 
                             if( checkValid(True) )
                                 {
-                                if( strlen(data) < maxLen )
+                                if( strlen(data) + len <= maxLen )
                                     {
                                     if( firstPos > curPos )
                                         firstPos = curPos;
-                                    memmove( data+curPos+1, data+curPos, strlen(data+curPos)+1 );
-                                    data[curPos++] = event.keyDown.charScan.charCode;
+                                    memmove( data+curPos+len, data+curPos, strlen(data+curPos)+1 );
+                                    memcpy( data+curPos, event.keyDown.text, len );
+                                    curPos += len;
                                     }
                                 checkValid(False);
                                 }
@@ -366,9 +379,10 @@ void TInputLine::handleEvent( TEvent& event )
                     selStart = 0;
                     selEnd = 0;
                     }
-                if( firstPos > curPos )
-                    firstPos = curPos;
-                i = curPos - size.x + 2;
+                curWidth = displayedPos(curPos);
+                if( firstPos > curWidth )
+                    firstPos = curWidth;
+                i = curWidth - size.x + 2;
                 if( firstPos < i )
                     firstPos = i;
                 drawView();
@@ -384,7 +398,7 @@ void TInputLine::selectAll( Boolean enable )
         curPos = selEnd = strlen(data);
     else
         curPos = selEnd = 0;
-    firstPos = max( 0, curPos-size.x+2 );
+    firstPos = max( 0, displayedPos(curPos)-size.x+2 );
     drawView();
 }