Keep characters after a truncated UTF-8 sequence (#3690)

A multibyte UTF-8 sequence cut short by a non-continuation byte took
every remaining byte of the string, so "a\xf0Zb" became "a\ufffd".

Replace only the lead byte and the continuation bytes that follow it,
the same as an invalid sequence that is not cut short, so "a\xf0Zb"
now becomes "a\ufffdZb". This also changes length and explode on such
strings, which now count the characters that used to be dropped.

src/jv_unicode.c

@@ -38,8 +38,11 @@ const char* jvp_utf8_next_multibyte(const char* in, const char* end, int* codepo
     /* Bad single byte - either an invalid byte or an out-of-place continuation byte */
     length = 1;
   } else if (length > end - in) {
-    /* String ends before UTF8 sequence ends */
-    length = end - in;
+    /* String ends before UTF8 sequence ends, so skip only its continuation bytes */
+    for (length = 1; in + length < end; length++) {
+      if (utf8_coding_length[(unsigned char)in[length]] != UTF8_CONTINUATION_BYTE)
+        break;
+    }
   } else {
     codepoint = ((unsigned)in[0]) & utf8_coding_bits[first];
     for (int i=1; i<length; i++) {

tests/jq.test

@@ -89,6 +89,11 @@ map(@base64d)
 ["YWJjZGVmZ2hpav9rbG1ub3A=", "YWJjZGVmZ2hpasOpa2xtbm9w5pel", "/2FiY2RlZmdoaWo=", "YWJjZGVmZ2hpauaX", "YWJjZGVmZ2hpau2ggHh5eg=="]
 ["abcdefghij�klmnop","abcdefghijéklmnop日","�abcdefghij","abcdefghij�","abcdefghij�xyz"]
 
+# Truncated UTF-8 sequence keeps the characters after it
+map(@base64d)
+["8Fo=", "4mE=", "8J9a", "YfBaYg==", "8J+Y"]
+["�Z","�a","�Z","a�Zb","�"]
+
 @uri
 "\u03bc"
 "%CE%BC"