[PATCH] Allow non-ASCII characters in Rust identifiers

[PATCH] Allow non-ASCII characters in Rust identifiers

[Tom Tromey]

Rust 1.53 (quite a while ago now) ungated the support for non-ASCII
identifiers.  This didn't work in gdb.  This is PR rust/20166.

This patch fixes the problem by allowing non-ASCII characters to be
considered as identifier components.  It seemed simplest to just pass
them through -- doing any extra checking didn't seem worthwhile.

The new test also verifies that such characters are allowed in strings
and character literals as well.  The latter also required a bit of
work in the lexer.

Bug: https://sourceware.org/bugzilla/show_bug.cgi?id=20166
---
 gdb/rust-parse.c                   | 70 ++++++++++++++++++++++--------
 gdb/testsuite/gdb.rust/unicode.exp | 51 ++++++++++++++++++++++
 gdb/testsuite/gdb.rust/unicode.rs  | 26 +++++++++++
 3 files changed, 129 insertions(+), 18 deletions(-)
 create mode 100644 gdb/testsuite/gdb.rust/unicode.exp
 create mode 100644 gdb/testsuite/gdb.rust/unicode.rs

diff --git a/gdb/rust-parse.c b/gdb/rust-parse.c
index 31a1ee3b38f..aa215f9cf2a 100644
--- a/gdb/rust-parse.c
+++ b/gdb/rust-parse.c
@@ -33,6 +33,12 @@
 
 using namespace expr;
 
+#if WORDS_BIGENDIAN
+#define UTF32 "UTF-32BE"
+#else
+#define UTF32 "UTF-32LE"
+#endif
+
 /* A regular expression for matching Rust numbers.  This is split up
    since it is very long and this gives us a way to comment the
    sections.  */
@@ -577,6 +583,35 @@ rust_parser::lex_escape (int is_byte)
   return result;
 }
 
+/* A helper for lex_character.  Search forward for the closing single
+   quote, then convert the bytes from the host charset to UTF-32.  */
+
+static uint32_t
+lex_multibyte_char (const char *text, int *len)
+{
+  /* Only look a maximum of 5 bytes for the closing quote.  This is
+     the maximum for UTF-8.  */
+  int quote;
+  gdb_assert (text[0] != '\'');
+  for (quote = 1; text[quote] != '\0' && text[quote] != '\''; ++quote)
+    ;
+  *len = quote;
+  /* The caller will issue an error.  */
+  if (text[quote] == '\0')
+    return 0;
+
+  auto_obstack result;
+  convert_between_encodings (host_charset (), UTF32, (const gdb_byte *) text,
+			     quote, 1, &result, translit_none);
+
+  int size = obstack_object_size (&result);
+  if (size > 4)
+    error (_("overlong character literal"));
+  uint32_t value;
+  memcpy (&value, obstack_finish (&result), size);
+  return value;
+}
+
 /* Lex a character constant.  */
 
 int
@@ -592,13 +627,15 @@ rust_parser::lex_character ()
     }
   gdb_assert (pstate->lexptr[0] == '\'');
   ++pstate->lexptr;
-  /* This should handle UTF-8 here.  */
-  if (pstate->lexptr[0] == '\\')
+  if (pstate->lexptr[0] == '\'')
+    error (_("empty character literal"));
+  else if (pstate->lexptr[0] == '\\')
     value = lex_escape (is_byte);
   else
     {
-      value = pstate->lexptr[0] & 0xff;
-      ++pstate->lexptr;
+      int len;
+      value = lex_multibyte_char (&pstate->lexptr[0], &len);
+      pstate->lexptr += len;
     }
 
   if (pstate->lexptr[0] != '\'')
@@ -695,16 +732,9 @@ rust_parser::lex_string ()
 	  if (is_byte)
 	    obstack_1grow (&obstack, value);
 	  else
-	    {
-#if WORDS_BIGENDIAN
-#define UTF32 "UTF-32BE"
-#else
-#define UTF32 "UTF-32LE"
-#endif
-	      convert_between_encodings (UTF32, "UTF-8", (gdb_byte *) &value,
-					 sizeof (value), sizeof (value),
-					 &obstack, translit_none);
-	    }
+	    convert_between_encodings (UTF32, "UTF-8", (gdb_byte *) &value,
+				       sizeof (value), sizeof (value),
+				       &obstack, translit_none);
 	}
       else if (pstate->lexptr[0] == '\0')
 	error (_("Unexpected EOF in string"));
@@ -746,7 +776,10 @@ rust_identifier_start_p (char c)
   return ((c >= 'a' && c <= 'z')
 	  || (c >= 'A' && c <= 'Z')
 	  || c == '_'
-	  || c == '$');
+	  || c == '$'
+	  /* Allow any non-ASCII character as an identifier.  There
+	     doesn't seem to be a need to be picky about this.  */
+	  || (c & 0x80) != 0);
 }
 
 /* Lex an identifier.  */
@@ -772,13 +805,14 @@ rust_parser::lex_identifier ()
 
   ++pstate->lexptr;
 
-  /* For the time being this doesn't handle Unicode rules.  Non-ASCII
-     identifiers are gated anyway.  */
+  /* Allow any non-ASCII character here.  This "handles" UTF-8 by
+     passing it through.  */
   while ((pstate->lexptr[0] >= 'a' && pstate->lexptr[0] <= 'z')
 	 || (pstate->lexptr[0] >= 'A' && pstate->lexptr[0] <= 'Z')
 	 || pstate->lexptr[0] == '_'
 	 || (is_gdb_var && pstate->lexptr[0] == '$')
-	 || (pstate->lexptr[0] >= '0' && pstate->lexptr[0] <= '9'))
+	 || (pstate->lexptr[0] >= '0' && pstate->lexptr[0] <= '9')
+	 || (pstate->lexptr[0] & 0x80) != 0)
     ++pstate->lexptr;
 
 
diff --git a/gdb/testsuite/gdb.rust/unicode.exp b/gdb/testsuite/gdb.rust/unicode.exp
new file mode 100644
index 00000000000..9de0a0e724f
--- /dev/null
+++ b/gdb/testsuite/gdb.rust/unicode.exp
@@ -0,0 +1,51 @@
+# Copyright (C) 2022 Free Software Foundation, Inc.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Test raw identifiers.
+
+load_lib rust-support.exp
+if {[skip_rust_tests]} {
+    continue
+}
+
+# Non-ASCII identifiers were allowed starting in 1.53.
+set v [split [rust_compiler_version] .]
+if {[lindex $v 0] == 1 && [lindex $v 1] < 53} {
+    untested "this test requires rust 1.53 or greater"
+    return -1
+}
+
+# Enable basic use of UTF-8.  LC_ALL gets reset for each testfile.
+setenv LC_ALL C.UTF-8
+
+standard_testfile .rs
+if {[prepare_for_testing "failed to prepare" $testfile $srcfile {debug rust}]} {
+    return -1
+}
+
+set line [gdb_get_line_number "set breakpoint here"]
+if {![runto ${srcfile}:$line]} {
+    untested "could not run to breakpoint"
+    return -1
+}
+
+gdb_test "print 𝕯" " = 98" "print D"
+gdb_test "print \"𝕯\"" " = \"𝕯\"" "print D in string"
+# This output is maybe not ideal, but it also isn't incorrect.
+gdb_test "print '𝕯'" " = 120175 '\\\\u\\\{01d56f\\\}'" \
+    "print D as char"
+gdb_test "print cç" " = 97" "print cc"
+
+gdb_test "print 'çc'" "overlong character literal" "print cc as char"
diff --git a/gdb/testsuite/gdb.rust/unicode.rs b/gdb/testsuite/gdb.rust/unicode.rs
new file mode 100644
index 00000000000..c6ca90e6450
--- /dev/null
+++ b/gdb/testsuite/gdb.rust/unicode.rs
@@ -0,0 +1,26 @@
+// Copyright (C) 2022 Free Software Foundation, Inc.
+
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#![allow(dead_code)]
+#![allow(unused_variables)]
+#![allow(unused_assignments)]
+#![allow(uncommon_codepoints)]
+#![allow(non_snake_case)]
+
+fn main() {
+    let 𝕯 = 98;
+    let cç = 97;
+    println!("{}, {}", 𝕯, cç);        // set breakpoint here
+}
-- 
2.31.1

Re: [PATCH] Allow non-ASCII characters in Rust identifiers

[Tom Tromey]

>>>>> "Tom" == Tom Tromey <tom@tromey.com> writes:

Tom> Rust 1.53 (quite a while ago now) ungated the support for non-ASCII
Tom> identifiers.  This didn't work in gdb.  This is PR rust/20166.

I'm checking this in.

Tom

Re: [PATCH] Allow non-ASCII characters in Rust identifiers

[Andrew Burgess via Gdb-patches]

Tom Tromey <tom@tromey.com> writes:

>>>>>> "Tom" == Tom Tromey <tom@tromey.com> writes:
>
> Tom> Rust 1.53 (quite a while ago now) ungated the support for non-ASCII
> Tom> identifiers.  This didn't work in gdb.  This is PR rust/20166.
>
> I'm checking this in.

Hi Tom!

I'm seeing this test fail.

 $ rustc --version
 rustc 1.59.0 (9d1b2106e 2022-02-23)

I've tested with gdb commit a723766c0e2 and 5187219460c.

Here's the output from gdb.log:

  (gdb) run 
  Starting program: /home/andrew/projects/binutils-gdb/build/gdb/testsuite/outputs/gdb.rust/unicode/unicode 
  [Thread debugging using libthread_db enabled]
  Using host libthread_db library "/lib64/libthread_db.so.1".
  
  Breakpoint 1, unicode::main () at /home/andrew/projects/binutils-gdb/build/gdb/testsuite/../../../src/gdb/testsuite/gdb.rust/unicode.rs:25
  25          println!("{}, {}", ð<U+009D><U+0095>¯, cç);        // set breakpoint here
  (gdb) print ð<U+009D><U+0095>¯
  unexpected token
  (gdb) FAIL: gdb.rust/unicode.exp: print D
  print "ð<U+009D><U+0095>¯"
  $1 = "ð\302\235\302\225¯"
  (gdb) FAIL: gdb.rust/unicode.exp: print D in string
  print 'ð<U+009D><U+0095>¯'
  Unterminated character literal
  (gdb) FAIL: gdb.rust/unicode.exp: print D as char
  print cç
  No symbol 'c' in current context
  (gdb) FAIL: gdb.rust/unicode.exp: print cc
  print 'çc'
  Unterminated character literal
  (gdb) FAIL: gdb.rust/unicode.exp: print cc as char

Do these pass for you?  Any suggestions for where to start looking?

Thanks,
Andrew

Re: [PATCH] Allow non-ASCII characters in Rust identifiers

[Tom Tromey]

Andrew> I'm seeing this test fail.

Andrew>  $ rustc --version
Andrew>  rustc 1.59.0 (9d1b2106e 2022-02-23)

I installed this version with "rustup toolchain install 1.59.0" and set
it to be my default.

Andrew> I've tested with gdb commit a723766c0e2 and 5187219460c.

I tried 552f1157c6262, a recent-ish git master.
It works fine for me.

Andrew> Do these pass for you?  Any suggestions for where to start looking?

I wonder if this line in the .exp isn't having the desired effect:

    setenv LC_ALL C.UTF-8

Is this happening interactively or in some kind of automation
environment?  Are the correct locales installed?  Do other
LC_ALL-setting tests fail?

Andrew>  print "ð<U+009D><U+0095>¯"
Andrew>  $1 = "ð\302\235\302\225¯"

One thing I'd suggest is checking by hand if either the 'print' line or
the '$1 = ' line has the correct byte values for the UTF-8 encoded form
of the character in question.

Tom

Re: [PATCH] Allow non-ASCII characters in Rust identifiers

[Andrew Burgess via Gdb-patches]

Tom Tromey <tom@tromey.com> writes:

> Andrew> I'm seeing this test fail.
>
> Andrew>  $ rustc --version
> Andrew>  rustc 1.59.0 (9d1b2106e 2022-02-23)
>
> I installed this version with "rustup toolchain install 1.59.0" and set
> it to be my default.
>
> Andrew> I've tested with gdb commit a723766c0e2 and 5187219460c.
>
> I tried 552f1157c6262, a recent-ish git master.
> It works fine for me.
>
> Andrew> Do these pass for you?  Any suggestions for where to start looking?
>
> I wonder if this line in the .exp isn't having the desired effect:
>
>     setenv LC_ALL C.UTF-8
>
> Is this happening interactively or in some kind of automation
> environment?  Are the correct locales installed?  Do other
> LC_ALL-setting tests fail?

This is when I run under dejagnu.  If I run the test manually, and copy
the commands from the .exp file by hand, pasting them into my GDB
session, it all appears to work fine.

I'm not sure how I'd check if the correct locales are installed (I mean,
I'm not sure what I'd be looking for), but I guess as it passes when run
manually, then I'm probably OK.

Looking for scripts that set or mention LC_ALL, I found these:

  gdb.base/utf8-identifiers.exp
  gdb.python/py-source-styling.exp
  gdb.ada/non-ascii-utf-8.exp
  gdb.ada/non-ascii-latin-3.exp
  gdb.ada/non-ascii-latin-1.exp

These all run fine, except for 3 failures in
gdb.ada/non-ascii-utf-8.exp, which look suspiciously similar:

  print VAR_ð<U+0090><U+0090><U+0081>
  No definition of "var_ð<U+0090><U+0090><U+0081>" in current context.
  (gdb) FAIL: gdb.ada/non-ascii-utf-8.exp: print VAR_ð<U+0090><U+0090><U+0081>
  print var_ð<U+0090><U+0090>©
  No definition of "var_ð<U+0090><U+0090>©" in current context.
  (gdb) FAIL: gdb.ada/non-ascii-utf-8.exp: print var_ð<U+0090><U+0090>©
  ... snip ...
  break FUNC_ð<U+0090><U+0090><U+0081>
  Function "FUNC_ð<U+0090><U+0090><U+0081>" not defined.
  Make breakpoint pending on future shared library load? (y or [n]) n
  (gdb) FAIL: gdb.ada/non-ascii-utf-8.exp: setting breakpoint at FUNC_ð<U+0090><U+0090><U+0081>

>
> Andrew>  print "ð<U+009D><U+0095>¯"
> Andrew>  $1 = "ð\302\235\302\225¯"
>
> One thing I'd suggest is checking by hand if either the 'print' line or
> the '$1 = ' line has the correct byte values for the UTF-8 encoded form
> of the character in question.

So, this is weird.  When I look at the .exp file, I see the bytes of the
unicode character as 0xf0 0x9f 0x95 0xaf, which looks correct:

  https://www.fileformat.info/info/unicode/char/1d56f/index.htm

But, when I look at the gdb.log file, I see the following bytes 0xc3
0xb0 0xc2 0x9d 0xc2 0x95 0xc2 0xaf.

Compared to the original, the first '0xf0' changes to '0xc3 0xb0', while
all the subequent bytes get a 0xc2 byte before them.

Does any of this give any clues to what might be happening?

Thanks,
Andrew

Re: [PATCH] Allow non-ASCII characters in Rust identifiers

[Andrew Burgess via Gdb-patches]

Andrew Burgess <aburgess@redhat.com> writes:

> Tom Tromey <tom@tromey.com> writes:
>
>> Andrew> I'm seeing this test fail.
>>
>> Andrew>  $ rustc --version
>> Andrew>  rustc 1.59.0 (9d1b2106e 2022-02-23)
>>
>> I installed this version with "rustup toolchain install 1.59.0" and set
>> it to be my default.
>>
>> Andrew> I've tested with gdb commit a723766c0e2 and 5187219460c.
>>
>> I tried 552f1157c6262, a recent-ish git master.
>> It works fine for me.
>>
>> Andrew> Do these pass for you?  Any suggestions for where to start looking?
>>
>> I wonder if this line in the .exp isn't having the desired effect:
>>
>>     setenv LC_ALL C.UTF-8
>>
>> Is this happening interactively or in some kind of automation
>> environment?  Are the correct locales installed?  Do other
>> LC_ALL-setting tests fail?
>
> This is when I run under dejagnu.  If I run the test manually, and copy
> the commands from the .exp file by hand, pasting them into my GDB
> session, it all appears to work fine.
>
> I'm not sure how I'd check if the correct locales are installed (I mean,
> I'm not sure what I'd be looking for), but I guess as it passes when run
> manually, then I'm probably OK.
>
> Looking for scripts that set or mention LC_ALL, I found these:
>
>   gdb.base/utf8-identifiers.exp
>   gdb.python/py-source-styling.exp
>   gdb.ada/non-ascii-utf-8.exp
>   gdb.ada/non-ascii-latin-3.exp
>   gdb.ada/non-ascii-latin-1.exp
>
> These all run fine, except for 3 failures in
> gdb.ada/non-ascii-utf-8.exp, which look suspiciously similar:
>
>   print VAR_ð<U+0090><U+0090><U+0081>
>   No definition of "var_ð<U+0090><U+0090><U+0081>" in current context.
>   (gdb) FAIL: gdb.ada/non-ascii-utf-8.exp: print VAR_ð<U+0090><U+0090><U+0081>
>   print var_ð<U+0090><U+0090>©
>   No definition of "var_ð<U+0090><U+0090>©" in current context.
>   (gdb) FAIL: gdb.ada/non-ascii-utf-8.exp: print var_ð<U+0090><U+0090>©
>   ... snip ...
>   break FUNC_ð<U+0090><U+0090><U+0081>
>   Function "FUNC_ð<U+0090><U+0090><U+0081>" not defined.
>   Make breakpoint pending on future shared library load? (y or [n]) n
>   (gdb) FAIL: gdb.ada/non-ascii-utf-8.exp: setting breakpoint at FUNC_ð<U+0090><U+0090><U+0081>
>
>>
>> Andrew>  print "ð<U+009D><U+0095>¯"
>> Andrew>  $1 = "ð\302\235\302\225¯"
>>
>> One thing I'd suggest is checking by hand if either the 'print' line or
>> the '$1 = ' line has the correct byte values for the UTF-8 encoded form
>> of the character in question.
>
> So, this is weird.  When I look at the .exp file, I see the bytes of the
> unicode character as 0xf0 0x9f 0x95 0xaf, which looks correct:
>
>   https://www.fileformat.info/info/unicode/char/1d56f/index.htm
>
> But, when I look at the gdb.log file, I see the following bytes 0xc3
> 0xb0 0xc2 0x9d 0xc2 0x95 0xc2 0xaf.
>
> Compared to the original, the first '0xf0' changes to '0xc3 0xb0', while
> all the subequent bytes get a 0xc2 byte before them.
>
> Does any of this give any clues to what might be happening?

So I put this into a text file 'unicode.tcl':

  puts "print 𝕯"

(just in case that gets mangled in transit, that's the same unicode
character as is used in the gdb.rust/unicode.exp test)

and then I did:

  $ tclsh unicode.tcl

and I get the same corrupted bytes as I see from the test script (c3 b0
c2 9d c2 95 c2 af).  So the problem appears to be with my build of tcl.

I'm currently running tcl 8.6.  I wonder if you could compare this to
the behaviour of your tclsh.

Thanks,
Andrew

Re: [PATCH] Allow non-ASCII characters in Rust identifiers

[Tom Tromey]

Andrew> So I put this into a text file 'unicode.tcl':
Andrew>   puts "print 𝕯"
Andrew> (just in case that gets mangled in transit, that's the same unicode
Andrew> character as is used in the gdb.rust/unicode.exp test)

Andrew> I'm currently running tcl 8.6.  I wonder if you could compare this to
Andrew> the behaviour of your tclsh.

This works for me.  I'm using the system tclsh on Fedora 34.
tclsh doesn't seem to support --version, but:

prentzel. rpm -q tcl
tcl-8.6.10-5.fc34.x86_64

Andrew> Compared to the original, the first '0xf0' changes to '0xc3 0xb0', while
Andrew> all the subequent bytes get a 0xc2 byte before them.

Unicode U+00f0 is represented as 0xc3 0xb0 in UTF-8.  So one idea is if
tclsh thinks the input is Latin-1, where the code points generally map
identically to Unicode code points, then this conversion would be done
if converting from the file encoding to UTF-8.

That is, tclsh reads 0xf0, but thinking it is reading a Latin-1
character, converts that to the corresponding Uncode character U+00f0,
and from there to the bytes that are seen.

I have LANG=en_US.UTF-8, which may explain why my default encoding is
UTF-8.  Perhaps this setting is the problem - if you are running in a
Latin-1 locale then the .exp files will be recoded incorrectly as they
are read by the interpreter.

I don't know if there's a way to set the file encoding in a way that Tcl
will recognize.  We could maybe try a UTF-8 BOM.

Tom