Skip to content

Make parsing more relaxed #152

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Make parsing more relaxed #152

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3a95a53
Make parsing more relaxed
Jan 5, 2015
8686ac5
Update comment
Jan 5, 2015
0ed5ab3
Remove unnecessary codes
Jan 5, 2015
f6c7cdd
minor update
Jan 5, 2015
5083269
Update yajl_parser.c
Jan 5, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/api/yajl_common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,13 @@ typedef struct
void * ctx;
} yajl_alloc_funcs;

#ifdef YAJL_ALLOW_SINGLE_QUOTES
typedef enum {
yajl_double_quote = 1,
yajl_single_quote
} yajl_quote_type;
#endif

#ifdef __cplusplus
}
#endif
Expand Down
7 changes: 7 additions & 0 deletions src/api/yajl_gen.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,9 +128,16 @@ extern "C" {
YAJL_API yajl_gen_status yajl_gen_number(yajl_gen hand,
const char * num,
size_t len);
#ifdef YAJL_ALLOW_SINGLE_QUOTES
YAJL_API yajl_gen_status yajl_gen_string(yajl_gen hand,
const unsigned char * str,
size_t len,
yajl_quote_type quote);
#else
YAJL_API yajl_gen_status yajl_gen_string(yajl_gen hand,
const unsigned char * str,
size_t len);
#endif
YAJL_API yajl_gen_status yajl_gen_null(yajl_gen hand);
YAJL_API yajl_gen_status yajl_gen_bool(yajl_gen hand, int boolean);
YAJL_API yajl_gen_status yajl_gen_map_open(yajl_gen hand);
Expand Down
16 changes: 15 additions & 1 deletion src/api/yajl_parse.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,12 +83,22 @@ extern "C" {

/** strings are returned as pointers into the JSON text when,
* possible, as a result, they are _not_ null padded */
#ifdef YAJL_ALLOW_SINGLE_QUOTES
int (* yajl_string)(void * ctx, const unsigned char * stringVal,
size_t stringLen, yajl_quote_type quote);
#else
int (* yajl_string)(void * ctx, const unsigned char * stringVal,
size_t stringLen);
#endif

int (* yajl_start_map)(void * ctx);
#ifdef YAJL_ALLOW_SINGLE_QUOTES
int (* yajl_map_key)(void * ctx, const unsigned char * key,
size_t stringLen, yajl_quote_type quote);
#else
int (* yajl_map_key)(void * ctx, const unsigned char * key,
size_t stringLen);
#endif
int (* yajl_end_map)(void * ctx);

int (* yajl_start_array)(void * ctx);
Expand Down Expand Up @@ -156,7 +166,11 @@ extern "C" {
* yajl will enter an error state (premature EOF). Setting this
* flag suppresses that check and the corresponding error.
*/
yajl_allow_partial_values = 0x10
yajl_allow_partial_values = 0x10,
/**
* Allow a comma trailing in the last element of array (or map)
*/
yajl_allow_trailing_separator = 0x20
} yajl_option;

/** allow the modification of parser options subsequent to handle
Expand Down
3 changes: 2 additions & 1 deletion src/yajl.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ yajl_alloc(const yajl_callbacks * callbacks,
hand->lexer = NULL;
hand->bytesConsumed = 0;
hand->decodeBuf = yajl_buf_alloc(&(hand->alloc));
hand->flags = 0;
hand->flags = 0;
yajl_bs_init(hand->stateStack, &(hand->alloc));
yajl_bs_push(hand->stateStack, yajl_state_start);

Expand All @@ -91,6 +91,7 @@ yajl_config(yajl_handle h, yajl_option opt, ...)
case yajl_allow_trailing_garbage:
case yajl_allow_multiple_values:
case yajl_allow_partial_values:
case yajl_allow_trailing_separator:
if (va_arg(ap, int)) h->flags |= opt;
else h->flags &= ~opt;
break;
Expand Down
6 changes: 6 additions & 0 deletions src/yajl_encode.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@ yajl_string_encode(const yajl_print_t print,
*/
case '/': if (escape_solidus) escaped = "\\/"; break;
case '"': escaped = "\\\""; break;
#ifdef YAJL_ALLOW_SINGLE_QUOTES
case '\'': escaped = "\\'"; break;
#endif
case '\f': escaped = "\\f"; break;
case '\b': escaped = "\\b"; break;
case '\t': escaped = "\\t"; break;
Expand Down Expand Up @@ -130,6 +133,9 @@ void yajl_string_decode(yajl_buf buf, const unsigned char * str,
case '\\': unescaped = "\\"; break;
case '/': unescaped = "/"; break;
case '"': unescaped = "\""; break;
#ifdef YAJL_ALLOW_SINGLE_QUOTES
case '\'': unescaped = "\'"; break;
#endif
case 'f': unescaped = "\f"; break;
case 'b': unescaped = "\b"; break;
case 't': unescaped = "\t"; break;
Expand Down
18 changes: 13 additions & 5 deletions src/yajl_gen.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,8 +251,11 @@ yajl_gen_number(yajl_gen g, const char * s, size_t l)
}

yajl_gen_status
yajl_gen_string(yajl_gen g, const unsigned char * str,
size_t len)
yajl_gen_string(yajl_gen g, const unsigned char * str, size_t len
#ifdef YAJL_ALLOW_SINGLE_QUOTES
, yajl_quote_type quote
#endif
)
{
// if validation is enabled, check that the string is valid utf8
// XXX: This checking could be done a little faster, in the same pass as
Expand All @@ -263,9 +266,14 @@ yajl_gen_string(yajl_gen g, const unsigned char * str,
}
}
ENSURE_VALID_STATE; INSERT_SEP; INSERT_WHITESPACE;
g->print(g->ctx, "\"", 1);
#ifdef YAJL_ALLOW_SINGLE_QUOTES
char *qstr = (quote == yajl_single_quote) ? "'" : "\"";
#else
static char * const qstr = "\"";
#endif
g->print(g->ctx, qstr, 1);
yajl_string_encode(g->print, g->ctx, str, len, g->flags & yajl_gen_escape_solidus);
g->print(g->ctx, "\"", 1);
g->print(g->ctx, qstr, 1);
APPENDED_ATOM;
FINAL_NEWLINE;
return yajl_gen_status_ok;
Expand All @@ -286,7 +294,7 @@ yajl_gen_bool(yajl_gen g, int boolean)
{
const char * val = boolean ? "true" : "false";

ENSURE_VALID_STATE; ENSURE_NOT_KEY; INSERT_SEP; INSERT_WHITESPACE;
ENSURE_VALID_STATE; ENSURE_NOT_KEY; INSERT_SEP; INSERT_WHITESPACE;
g->print(g->ctx, val, (unsigned int)strlen(val));
APPENDED_ATOM;
FINAL_NEWLINE;
Expand Down
62 changes: 58 additions & 4 deletions src/yajl_lex.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,11 @@ struct yajl_lexer_t {
unsigned int validateUTF8;

yajl_alloc_funcs * alloc;

#ifdef YAJL_ALLOW_SINGLE_QUOTES
/* current quote type */
yajl_quote_type quote;
#endif
};

#define readChar(lxr, txt, off) \
Expand All @@ -100,6 +105,17 @@ struct yajl_lexer_t {

#define unreadChar(lxr, off) ((*(off) > 0) ? (*(off))-- : ((lxr)->bufOff--))

#ifdef YAJL_ALLOW_SINGLE_QUOTES
#define isQuote(ch) (ch == '"' || ch == '\'')
#define quoteType(ch) ((ch == '"') ? yajl_double_quote : yajl_single_quote)
#endif

#ifdef YAJL_ALLOW_SINGLE_QUOTES
#define isClosingQuote(ch) (isQuote(ch) && quoteType(ch) == lexer->quote)
#else
#define isClosingQuote(ch) (ch == '"')
#endif

yajl_lexer
yajl_lex_alloc(yajl_alloc_funcs * alloc,
unsigned int allowComments, unsigned int validateUTF8)
Expand Down Expand Up @@ -141,8 +157,11 @@ static const char charLookupTable[256] =
/*08*/ IJC , IJC , IJC , IJC , IJC , IJC , IJC , IJC ,
/*10*/ IJC , IJC , IJC , IJC , IJC , IJC , IJC , IJC ,
/*18*/ IJC , IJC , IJC , IJC , IJC , IJC , IJC , IJC ,

/*20*/ 0 , 0 , NFP|VEC|IJC, 0 , 0 , 0 , 0 , 0 ,
#ifdef YAJL_ALLOW_SINGLE_QUOTES
/*20*/ 0 , 0 , NFP|VEC|IJC, 0 , 0 , 0 , 0 , NFP|VEC|IJC,
#else
/*20*/ 0 , 0 , NFP|VEC|IJC, 0 , 0 , 0 , 0 , 0,
#endif
/*28*/ 0 , 0 , 0 , 0 , 0 , 0 , 0 , VEC ,
/*30*/ VHC , VHC , VHC , VHC , VHC , VHC , VHC , VHC ,
/*38*/ VHC , VHC , 0 , 0 , 0 , 0 , 0 , 0 ,
Expand Down Expand Up @@ -252,11 +271,20 @@ if (*offset >= jsonTextLen) { \
* be skipped.
* (lth) hi world, any thoughts on how to make this routine faster? */
static size_t
yajl_string_scan(const unsigned char * buf, size_t len, int utf8check)
yajl_string_scan(const unsigned char * buf, size_t len, int utf8check
#ifdef YAJL_ALLOW_SINGLE_QUOTES
, yajl_quote_type quote
#endif
)
{
unsigned char mask = IJC|NFP|(utf8check ? NUC : 0);
size_t skip = 0;
#ifdef YAJL_ALLOW_SINGLE_QUOTES
while (skip < len && (!(charLookupTable[*buf] & mask) ||
(isQuote(*buf) && quoteType(*buf) != quote)))
#else
while (skip < len && !(charLookupTable[*buf] & mask))
#endif
{
skip++;
buf++;
Expand Down Expand Up @@ -286,13 +314,23 @@ yajl_lex_string(yajl_lexer lexer, const unsigned char * jsonText,
p = ((const unsigned char *) yajl_buf_data(lexer->buf) +
(lexer->bufOff));
len = yajl_buf_len(lexer->buf) - lexer->bufOff;
#ifdef YAJL_ALLOW_SINGLE_QUOTES
lexer->bufOff += yajl_string_scan(p, len, lexer->validateUTF8,
lexer->quote);
#else
lexer->bufOff += yajl_string_scan(p, len, lexer->validateUTF8);
#endif
}
else if (*offset < jsonTextLen)
{
p = jsonText + *offset;
len = jsonTextLen - *offset;
#ifdef YAJL_ALLOW_SINGLE_QUOTES
*offset += yajl_string_scan(p, len, lexer->validateUTF8,
lexer->quote);
#else
*offset += yajl_string_scan(p, len, lexer->validateUTF8);
#endif
}
}

Expand All @@ -301,7 +339,7 @@ yajl_lex_string(yajl_lexer lexer, const unsigned char * jsonText,
curChar = readChar(lexer, jsonText, offset);

/* quote terminates */
if (curChar == '"') {
if (isClosingQuote(curChar)) {
tok = yajl_tok_string;
break;
}
Expand Down Expand Up @@ -510,6 +548,9 @@ yajl_lex_lex(yajl_lexer lexer, const unsigned char * jsonText,
for (;;) {
assert(*offset <= jsonTextLen);

#ifdef YAJL_ALLOW_SINGLE_QUOTES
lexer->quote = 0;
#endif
if (*offset >= jsonTextLen) {
tok = yajl_tok_eof;
goto lexed;
Expand Down Expand Up @@ -593,7 +634,13 @@ yajl_lex_lex(yajl_lexer lexer, const unsigned char * jsonText,
tok = yajl_tok_null;
goto lexed;
}
#ifdef YAJL_ALLOW_SINGLE_QUOTES
case '\'':
#endif
case '"': {
#ifdef YAJL_ALLOW_SINGLE_QUOTES
lexer->quote = quoteType(c);
#endif
tok = yajl_lex_string(lexer, (const unsigned char *) jsonText,
jsonTextLen, offset);
goto lexed;
Expand Down Expand Up @@ -761,3 +808,10 @@ yajl_tok yajl_lex_peek(yajl_lexer lexer, const unsigned char * jsonText,

return tok;
}

#ifdef YAJL_ALLOW_SINGLE_QUOTES
yajl_quote_type yajl_lex_current_quote(yajl_lexer lexer)
{
return lexer->quote;
}
#endif
4 changes: 4 additions & 0 deletions src/yajl_lex.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,4 +114,8 @@ size_t yajl_lex_current_line(yajl_lexer lexer);
* \n or \r */
size_t yajl_lex_current_char(yajl_lexer lexer);

#ifdef YAJL_ALLOW_SINGLE_QUOTES
yajl_quote_type yajl_lex_current_quote(yajl_lexer lexer);
#endif

#endif
39 changes: 30 additions & 9 deletions src/yajl_parser.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,17 +240,30 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText,
goto around_again;
case yajl_tok_string:
if (hand->callbacks && hand->callbacks->yajl_string) {
#ifdef YAJL_ALLOW_SINGLE_QUOTES
yajl_quote_type q = yajl_lex_current_quote(hand->lexer);
_CC_CHK(hand->callbacks->yajl_string(hand->ctx,
buf, bufLen, q));
#else
_CC_CHK(hand->callbacks->yajl_string(hand->ctx,
buf, bufLen));
#endif
}
break;
case yajl_tok_string_with_escapes:
if (hand->callbacks && hand->callbacks->yajl_string) {
yajl_buf_clear(hand->decodeBuf);
yajl_string_decode(hand->decodeBuf, buf, bufLen);
#ifdef YAJL_ALLOW_SINGLE_QUOTES
yajl_quote_type q = yajl_lex_current_quote(hand->lexer);
_CC_CHK(hand->callbacks->yajl_string(
hand->ctx, yajl_buf_data(hand->decodeBuf),
yajl_buf_len(hand->decodeBuf), q));
#else
_CC_CHK(hand->callbacks->yajl_string(
hand->ctx, yajl_buf_data(hand->decodeBuf),
yajl_buf_len(hand->decodeBuf)));
#endif
}
break;
case yajl_tok_bool:
Expand Down Expand Up @@ -284,7 +297,7 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText,
} else if (hand->callbacks->yajl_integer) {
long long int i = 0;
errno = 0;
i = yajl_parse_integer(buf, bufLen);
i = yajl_parse_integer(buf, (unsigned int)bufLen);
if ((i == LLONG_MIN || i == LLONG_MAX) &&
errno == ERANGE)
{
Expand Down Expand Up @@ -331,12 +344,12 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText,
}
break;
case yajl_tok_right_brace: {
if (yajl_bs_current(hand->stateStack) ==
yajl_state_array_start)
yajl_state s = yajl_bs_current(hand->stateStack);
if (s == yajl_state_array_start ||
(s == yajl_state_array_need_val &&
(hand->flags & yajl_allow_trailing_separator)))
{
if (hand->callbacks &&
hand->callbacks->yajl_end_array)
{
if (hand->callbacks && hand->callbacks->yajl_end_array) {
_CC_CHK(hand->callbacks->yajl_end_array(hand->ctx));
}
yajl_bs_pop(hand->stateStack);
Expand Down Expand Up @@ -396,21 +409,29 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText,
/* intentional fall-through */
case yajl_tok_string:
if (hand->callbacks && hand->callbacks->yajl_map_key) {
#ifdef YAJL_ALLOW_SINGLE_QUOTES
yajl_quote_type q = yajl_lex_current_quote(hand->lexer);
_CC_CHK(hand->callbacks->yajl_map_key(hand->ctx, buf,
bufLen, q));
#else
_CC_CHK(hand->callbacks->yajl_map_key(hand->ctx, buf,
bufLen));
#endif
}
yajl_bs_set(hand->stateStack, yajl_state_map_sep);
goto around_again;
case yajl_tok_right_bracket:
if (yajl_bs_current(hand->stateStack) ==
yajl_state_map_start)
case yajl_tok_right_bracket: {
yajl_state s = yajl_bs_current(hand->stateStack);
if (!(s == yajl_state_map_need_key &&
!(hand->flags & yajl_allow_trailing_separator)))
{
if (hand->callbacks && hand->callbacks->yajl_end_map) {
_CC_CHK(hand->callbacks->yajl_end_map(hand->ctx));
}
yajl_bs_pop(hand->stateStack);
goto around_again;
}
}
default:
yajl_bs_set(hand->stateStack, yajl_state_parse_error);
hand->parseError =
Expand Down
6 changes: 5 additions & 1 deletion src/yajl_tree.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,11 @@ static int context_add_value (context_t *ctx, yajl_val v)
}

static int handle_string (void *ctx,
const unsigned char *string, size_t string_length)
const unsigned char *string, size_t string_length
#ifdef YAJL_ALLOW_SINGLE_QUOTES
, yajl_quote_type quote
#endif
)
{
yajl_val v;

Expand Down