Strip UTF-8 BOM at the start of each input file (#3657)
The UTF-8 BOM was stripped only at the start of the parser stream, but jq feeds all input files into a single parser, so a BOM at the start of the second and later files reached the parser as content and failed with a confusing "Invalid numeric literal" error. Strip it in the file readers instead, at the start of each input file including stdin, and in jv_load_file. This also strips the BOM for raw input, so -R and --rawfile no longer yield a leading U+FEFF, and a BOM at the start of a -f program file or an imported module is now accepted rather than reported as a syntax error. In exchange, "\uFEFF1" | fromjson is now a parse error, since a U+FEFF inside a string is content rather than an encoding marker, and the dedicated "Malformed BOM" error is gone.
This commit is contained in:
1 parent
17b87d25c7
commit
e001deb4cf
5 files changed
+34
-35
No files matched your search
+13
-2
@@ -43,6 +43,7 @@ jv jv_load_file(const char* filename, int raw) {
|
||||
// buffer, we need to be able to read the remainder of a sequence and add that
|
||||
// before appending.
|
||||
char buf[4096 + 4];
|
||||
int at_file_start = 1;
|
||||
while (!feof(file) && !ferror(file)) {
|
||||
size_t n = fread(buf, 1, sizeof(buf) - 4, file);
|
||||
if (n == 0)
|
||||
@@ -55,12 +56,22 @@ jv jv_load_file(const char* filename, int raw) {
|
||||
n += fread(end, 1, len, file);
|
||||
}
|
||||
|
||||
char *p = buf;
|
||||
if (at_file_start) {
|
||||
// Strip UTF-8 BOM at the start of the file
|
||||
if (n >= 3 && !memcmp(buf, "\xEF\xBB\xBF", 3)) {
|
||||
p += 3;
|
||||
n -= 3;
|
||||
}
|
||||
at_file_start = 0;
|
||||
}
|
||||
|
||||
if (raw) {
|
||||
data = jv_string_append_buf(data, buf, n);
|
||||
data = jv_string_append_buf(data, p, n);
|
||||
if (!jv_is_valid(data))
|
||||
break;
|
||||
} else {
|
||||
jv_parser_set_buf(parser, buf, n, !feof(file));
|
||||
jv_parser_set_buf(parser, p, n, !feof(file));
|
||||
jv value;
|
||||
while (jv_is_valid((value = jv_parser_next(parser))))
|
||||
data = jv_array_append(data, value);
|
||||
|
||||
Reference in new issue
Block a user