[PATCH v3] Fix crash when target-wide-charset doesn't match wchar_t size

[PATCH v3] Fix crash when target-wide-charset doesn't match wchar_t size

[Aditya Vidyadhar Kamath]

From: Aditya Vidyadhar Kamath <aditya.kamath1@ibm.com>

Tom asked [1] whether setting target-wide-charset to an encoding with
the wrong unit size can crash GDB.  It can.

First I compiled gdb/testsuite/gdb.base/wchar.c

      gcc -g -maix64 -o /tmp/wchar64 gdb/testsuite/gdb.base/wchar.c

Then debug with GDB and we get the crash

      (gdb) b wchar.c:44
      (gdb) run
      (gdb) set target-wide-charset UTF-16
      (gdb) print single

      Fatal signal: Segmentation fault
      ----- Backtrace -----

On 64-bit AIX wchar_t is 4 bytes wide.  Setting target-wide-charset
to UTF-16 causes wchar_iterator::iterate() to be constructed with
m_width=4 but a 2-byte-per-unit charset.  On the EILSEQ path, iconv
consumes only 2 bytes before returning, and then:

  m_input += m_width;   /* advances by 4 */
  m_bytes -= m_width;   /* m_bytes was 4, wraps to a huge size_t */

The loop races past the end of the buffer, corrupting the stack and
crashing inside libiconv.

Two fixes:

1. validate() now checks the unit width implied by the charset name
   against wchar_bit and rejects the setting with a clear error.
   The previous good value is restored on failure since the setshow
   framework stores the new value before calling the sfunc.

2. wchar_iterator::iterate() guards the EILSEQ path: if
   m_bytes < m_width, clamp m_bytes to zero instead of subtracting,
   so the loop terminates cleanly regardless of how the iterator
   was constructed.

With the patch, the same session now
gives:

      (gdb) set target-wide-charset UTF-16
      Character set `UTF-16' has a 2-byte character width but `wchar_t' on this target is 4 bytes wide.
      Use `set target-wide-charset auto' to let GDB choose the correct encoding automatically.
      (gdb) print single
      $1 = 48879 L'\xbeef'

Add tests to wchar.exp for this.

Tested on powerpc64-ibm-aix7.2.0.0 with 32-bit and 64-bit inferiors.

[1]: https://sourceware.org/pipermail/gdb-patches/2026-October/231110.html
---
 gdb/charset.c                    | 68 ++++++++++++++++++++++++++++++--
 gdb/testsuite/gdb.base/wchar.exp | 34 ++++++++++++++++
 2 files changed, 99 insertions(+), 3 deletions(-)

diff --git a/gdb/charset.c b/gdb/charset.c
index 1b0725b5078..327e4f4cf91 100644
--- a/gdb/charset.c
+++ b/gdb/charset.c
@@ -256,6 +256,10 @@ show_target_charset_name (struct ui_file *file, int from_tty,
 }
 
 static const char *target_wide_charset_name = "auto";
+
+/* Last known-good value of target_wide_charset_name, used to roll back
+   if a new setting fails validation.  */
+static const char *target_wide_charset_name_validated = "auto";
 static void
 show_target_wide_charset_name (struct ui_file *file,
 			       int from_tty,
@@ -336,6 +340,21 @@ set_be_le_names (struct gdbarch *gdbarch)
 /* 'Set charset', 'set host-charset', 'set target-charset', 'set
    target-wide-charset', 'set charset' sfunc's.  */
 
+/* Return the character width in bytes implied by a wide charset name:
+   2 for UTF-16/UCS-2, 4 for UTF-32/UCS-4, 0 if unknown.  */
+
+static int
+wide_charset_width (const char *name)
+{
+  if (strncasecmp (name, "UTF-16", 6) == 0
+      || strncasecmp (name, "UCS-2", 5) == 0)
+    return 2;
+  if (strncasecmp (name, "UTF-32", 6) == 0
+      || strncasecmp (name, "UCS-4", 5) == 0)
+    return 4;
+  return 0;
+}
+
 static void
 validate (struct gdbarch *gdbarch)
 {
@@ -346,6 +365,25 @@ validate (struct gdbarch *gdbarch)
 
   if (streq (target_wide_cset, "auto"))
     target_wide_cset = gdbarch_auto_wide_charset (gdbarch);
+  else
+    {
+      /* Reject charsets whose unit size doesn't match wchar_t.
+	 A mismatch (e.g. UTF-16 with a 4-byte wchar_t) makes iconv
+	 consume the wrong number of bytes and can crash GDB.  */
+      int charset_width = wide_charset_width (target_wide_cset);
+      if (charset_width != 0)
+	{
+	  int wchar_bytes = gdbarch_wchar_bit (gdbarch) / 8;
+	  if (charset_width != wchar_bytes)
+	    error (_ ("Character set \"%s\" has a %d-byte character width "
+		      "but \"wchar_t\" on this target is %d bytes wide.\n"
+		      "Use \"%ps\" to let GDB "
+		      "choose the correct encoding automatically."),
+		   target_wide_cset, charset_width, wchar_bytes,
+		   styled_string (command_style.style (),
+				  "set target-wide-charset auto"));
+	}
+    }
 
   desc = iconv_open (target_wide_cset, host_cset);
   if (desc == (iconv_t) -1)
@@ -395,7 +433,18 @@ static void
 set_target_wide_charset_sfunc (const char *charset, int from_tty,
 			       struct cmd_list_element *c)
 {
-  validate (get_current_arch ());
+  /* The setshow framework writes the new value before calling us, so
+     restore the previous setting if validation fails.  */
+  try
+    {
+      validate (get_current_arch ());
+      target_wide_charset_name_validated = target_wide_charset_name;
+    }
+  catch (const gdb_exception_error &)
+    {
+      target_wide_charset_name = target_wide_charset_name_validated;
+      throw;
+    }
 }
 
 /* sfunc for the 'show charset' command.  */
@@ -646,8 +695,21 @@ wchar_iterator::iterate (enum wchar_iterate_result *out_result,
 	      *out_result = wchar_iterate_invalid;
 	      *ptr = m_input;
 	      *len = m_width;
-	      m_input += m_width;
-	      m_bytes -= m_width;
+	      /* Guard against unsigned wrap: m_bytes < m_width can happen
+		 when the charset unit size doesn't match m_width (e.g. a
+		 UTF-16 charset used with a 4-byte wchar_t).  Clamp to zero
+		 rather than letting m_bytes underflow to a huge value and
+		 sending the loop past the end of the buffer.  */
+	      if (m_bytes >= m_width)
+		{
+		  m_input += m_width;
+		  m_bytes -= m_width;
+		}
+	      else
+		{
+		  m_input += m_bytes;
+		  m_bytes = 0;
+		}
 	      return 0;
 
 	    case E2BIG:
diff --git a/gdb/testsuite/gdb.base/wchar.exp b/gdb/testsuite/gdb.base/wchar.exp
index b02f4a78336..801aeb9ee20 100644
--- a/gdb/testsuite/gdb.base/wchar.exp
+++ b/gdb/testsuite/gdb.base/wchar.exp
@@ -75,3 +75,37 @@ gdb_test "printf \"%ls\\n\", 0" "\\(null\\)"
 # From PR exp/33124 - a bug when converting escapes.
 set wbs {L'\\'}
 gdb_test "print $wbs" " = $decimal [string_to_regexp $wbs]"
+
+# A wide charset whose unit size doesn't match wchar_t must be rejected,
+# otherwise printing wide chars can crash GDB.
+# Derive good/bad charsets from the inferior's actual wchar_t size so the
+# test is correct for both 32-bit (2-byte wchar_t -> UTF-16) and 64-bit
+# (4-byte wchar_t -> UTF-32) targets.
+set wchar_bytes 0
+gdb_test_multiple "print sizeof(wchar_t)" "" {
+    -re " = 2.*$gdb_prompt $" { set wchar_bytes 2 }
+    -re " = 4.*$gdb_prompt $" { set wchar_bytes 4 }
+}
+if { $wchar_bytes == 2 } {
+    set good_cs UTF-16
+    set bad_cs  UTF-32
+} elseif { $wchar_bytes == 4 } {
+    set good_cs UTF-32
+    set bad_cs  UTF-16
+} else {
+    return
+}
+
+gdb_test "set target-wide-charset $bad_cs" \
+    "Character set \"$bad_cs\" has a .-byte character width.*" \
+    "reject target-wide-charset $bad_cs"
+
+# The rejected setting must not stick.
+gdb_test "print single" "= 48879 L'\\\\xbeef'" \
+    "print wchar_t after rejected target-wide-charset"
+
+gdb_test_no_output "set target-wide-charset $good_cs" \
+    "accept target-wide-charset $good_cs"
+
+gdb_test "print single" "= 48879 L'\\\\xbeef'" \
+    "print wchar_t with explicit target-wide-charset"
-- 
2.51.2