From 70c9dc516fc3b92f8fb9721dba2fde32642fdbd6 Mon Sep 17 00:00:00 2001 From: skrmc <70046367+skrmc@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:59:37 -0500 Subject: [PATCH] Keep characters after a truncated UTF-8 sequence (#3690) A multibyte UTF-8 sequence cut short by a non-continuation byte took every remaining byte of the string, so "a\xf0Zb" became "a\ufffd". Replace only the lead byte and the continuation bytes that follow it, the same as an invalid sequence that is not cut short, so "a\xf0Zb" now becomes "a\ufffdZb". This also changes length and explode on such strings, which now count the characters that used to be dropped. --- src/jv_unicode.c | 7 +++++-- tests/jq.test | 5 +++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/jv_unicode.c b/src/jv_unicode.c index 258684ea..3846e6e3 100644 --- a/src/jv_unicode.c +++ b/src/jv_unicode.c @@ -38,8 +38,11 @@ const char* jvp_utf8_next_multibyte(const char* in, const char* end, int* codepo /* Bad single byte - either an invalid byte or an out-of-place continuation byte */ length = 1; } else if (length > end - in) { - /* String ends before UTF8 sequence ends */ - length = end - in; + /* String ends before UTF8 sequence ends, so skip only its continuation bytes */ + for (length = 1; in + length < end; length++) { + if (utf8_coding_length[(unsigned char)in[length]] != UTF8_CONTINUATION_BYTE) + break; + } } else { codepoint = ((unsigned)in[0]) & utf8_coding_bits[first]; for (int i=1; i