OLD | NEW |
1 // Protocol Buffers - Google's data interchange format | 1 // Protocol Buffers - Google's data interchange format |
2 // Copyright 2008 Google Inc. All rights reserved. | 2 // Copyright 2008 Google Inc. All rights reserved. |
3 // https://developers.google.com/protocol-buffers/ | 3 // https://developers.google.com/protocol-buffers/ |
4 // | 4 // |
5 // Redistribution and use in source and binary forms, with or without | 5 // Redistribution and use in source and binary forms, with or without |
6 // modification, are permitted provided that the following conditions are | 6 // modification, are permitted provided that the following conditions are |
7 // met: | 7 // met: |
8 // | 8 // |
9 // * Redistributions of source code must retain the above copyright | 9 // * Redistributions of source code must retain the above copyright |
10 // notice, this list of conditions and the following disclaimer. | 10 // notice, this list of conditions and the following disclaimer. |
(...skipping 27 matching lines...) Expand all Loading... |
38 #include <memory> | 38 #include <memory> |
39 #ifndef _SHARED_PTR_H | 39 #ifndef _SHARED_PTR_H |
40 #include <google/protobuf/stubs/shared_ptr.h> | 40 #include <google/protobuf/stubs/shared_ptr.h> |
41 #endif | 41 #endif |
42 | 42 |
43 #include <google/protobuf/stubs/logging.h> | 43 #include <google/protobuf/stubs/logging.h> |
44 #include <google/protobuf/stubs/common.h> | 44 #include <google/protobuf/stubs/common.h> |
45 #include <google/protobuf/util/internal/object_writer.h> | 45 #include <google/protobuf/util/internal/object_writer.h> |
46 #include <google/protobuf/util/internal/json_escaping.h> | 46 #include <google/protobuf/util/internal/json_escaping.h> |
47 #include <google/protobuf/stubs/strutil.h> | 47 #include <google/protobuf/stubs/strutil.h> |
48 #include <google/protobuf/stubs/mathlimits.h> | |
49 | |
50 | 48 |
51 namespace google { | 49 namespace google { |
52 namespace protobuf { | 50 namespace protobuf { |
53 namespace util { | 51 namespace util { |
54 | 52 |
55 // Allow these symbols to be referenced as util::Status, util::error::* in | 53 // Allow these symbols to be referenced as util::Status, util::error::* in |
56 // this file. | 54 // this file. |
57 using util::Status; | 55 using util::Status; |
58 namespace error { | 56 namespace error { |
59 using util::error::INTERNAL; | 57 using util::error::INTERNAL; |
(...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
102 leftover_(), | 100 leftover_(), |
103 json_(), | 101 json_(), |
104 p_(), | 102 p_(), |
105 key_(), | 103 key_(), |
106 key_storage_(), | 104 key_storage_(), |
107 finishing_(false), | 105 finishing_(false), |
108 parsed_(), | 106 parsed_(), |
109 parsed_storage_(), | 107 parsed_storage_(), |
110 string_open_(0), | 108 string_open_(0), |
111 chunk_storage_(), | 109 chunk_storage_(), |
112 coerce_to_utf8_(false), | 110 coerce_to_utf8_(false) { |
113 allow_empty_null_(false), | |
114 loose_float_number_conversion_(false) { | |
115 // Initialize the stack with a single value to be parsed. | 111 // Initialize the stack with a single value to be parsed. |
116 stack_.push(VALUE); | 112 stack_.push(VALUE); |
117 } | 113 } |
118 | 114 |
119 JsonStreamParser::~JsonStreamParser() {} | 115 JsonStreamParser::~JsonStreamParser() {} |
120 | 116 |
121 | 117 |
122 util::Status JsonStreamParser::Parse(StringPiece json) { | 118 util::Status JsonStreamParser::Parse(StringPiece json) { |
123 StringPiece chunk = json; | 119 StringPiece chunk = json; |
124 // If we have leftovers from a previous chunk, append the new chunk to it | 120 // If we have leftovers from a previous chunk, append the new chunk to it |
(...skipping 149 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
274 return ParseNumber(); | 270 return ParseNumber(); |
275 case BEGIN_TRUE: | 271 case BEGIN_TRUE: |
276 return ParseTrue(); | 272 return ParseTrue(); |
277 case BEGIN_FALSE: | 273 case BEGIN_FALSE: |
278 return ParseFalse(); | 274 return ParseFalse(); |
279 case BEGIN_NULL: | 275 case BEGIN_NULL: |
280 return ParseNull(); | 276 return ParseNull(); |
281 case UNKNOWN: | 277 case UNKNOWN: |
282 return ReportUnknown("Expected a value."); | 278 return ReportUnknown("Expected a value."); |
283 default: { | 279 default: { |
284 if (allow_empty_null_ && IsEmptyNullAllowed(type)) { | |
285 return ParseEmptyNull(); | |
286 } | |
287 | |
288 // Special case for having been cut off while parsing, wait for more data. | 280 // Special case for having been cut off while parsing, wait for more data. |
289 // This handles things like 'fals' being at the end of the string, we | 281 // This handles things like 'fals' being at the end of the string, we |
290 // don't know if the next char would be e, completing it, or something | 282 // don't know if the next char would be e, completing it, or something |
291 // else, making it invalid. | 283 // else, making it invalid. |
292 if (!finishing_ && p_.length() < false_len) { | 284 if (!finishing_ && p_.length() < false_len) { |
293 return util::Status::CANCELLED; | 285 return util::Status::CANCELLED; |
294 } | 286 } |
295 return ReportFailure("Unexpected token."); | 287 return ReportFailure("Unexpected token."); |
296 } | 288 } |
297 } | 289 } |
298 } | 290 } |
299 | 291 |
300 util::Status JsonStreamParser::ParseString() { | 292 util::Status JsonStreamParser::ParseString() { |
301 util::Status result = ParseStringHelper(); | 293 util::Status result = ParseStringHelper(); |
302 if (result.ok()) { | 294 if (result.ok()) { |
303 ow_->RenderString(key_, parsed_); | 295 ow_->RenderString(key_, parsed_); |
304 key_ = StringPiece(); | 296 key_.clear(); |
305 parsed_ = StringPiece(); | 297 parsed_.clear(); |
306 parsed_storage_.clear(); | 298 parsed_storage_.clear(); |
307 } | 299 } |
308 return result; | 300 return result; |
309 } | 301 } |
310 | 302 |
311 util::Status JsonStreamParser::ParseStringHelper() { | 303 util::Status JsonStreamParser::ParseStringHelper() { |
312 // If we haven't seen the start quote, grab it and remember it for later. | 304 // If we haven't seen the start quote, grab it and remember it for later. |
313 if (string_open_ == 0) { | 305 if (string_open_ == 0) { |
314 string_open_ = *p_.data(); | 306 string_open_ = *p_.data(); |
315 GOOGLE_DCHECK(string_open_ == '\"' || string_open_ == '\''); | 307 GOOGLE_DCHECK(string_open_ == '\"' || string_open_ == '\''); |
316 Advance(); | 308 Advance(); |
317 } | 309 } |
318 // Track where we last copied data from so we can minimize copying. | 310 // Track where we last copied data from so we can minimize copying. |
319 const char* last = p_.data(); | 311 const char* last = p_.data(); |
320 while (!p_.empty()) { | 312 while (!p_.empty()) { |
321 const char* data = p_.data(); | 313 const char* data = p_.data(); |
322 if (*data == '\\') { | 314 if (*data == '\\') { |
323 // We're about to handle an escape, copy all bytes from last to data. | 315 // We're about to handle an escape, copy all bytes from last to data. |
324 if (last < data) { | 316 if (last < data) { |
325 parsed_storage_.append(last, data - last); | 317 parsed_storage_.append(last, data - last); |
| 318 last = data; |
326 } | 319 } |
327 // If we ran out of string after the \, cancel or report an error | 320 // If we ran out of string after the \, cancel or report an error |
328 // depending on if we expect more data later. | 321 // depending on if we expect more data later. |
329 if (p_.length() == 1) { | 322 if (p_.length() == 1) { |
330 if (!finishing_) { | 323 if (!finishing_) { |
331 return util::Status::CANCELLED; | 324 return util::Status::CANCELLED; |
332 } | 325 } |
333 return ReportFailure("Closing quote expected in string."); | 326 return ReportFailure("Closing quote expected in string."); |
334 } | 327 } |
335 // Parse a unicode escape if we found \u in the string. | 328 // Parse a unicode escape if we found \u in the string. |
(...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
371 continue; | 364 continue; |
372 } | 365 } |
373 // If we found the closing quote note it, advance past it, and return. | 366 // If we found the closing quote note it, advance past it, and return. |
374 if (*data == string_open_) { | 367 if (*data == string_open_) { |
375 // If we didn't copy anything, reuse the input buffer. | 368 // If we didn't copy anything, reuse the input buffer. |
376 if (parsed_storage_.empty()) { | 369 if (parsed_storage_.empty()) { |
377 parsed_ = StringPiece(last, data - last); | 370 parsed_ = StringPiece(last, data - last); |
378 } else { | 371 } else { |
379 if (last < data) { | 372 if (last < data) { |
380 parsed_storage_.append(last, data - last); | 373 parsed_storage_.append(last, data - last); |
| 374 last = data; |
381 } | 375 } |
382 parsed_ = StringPiece(parsed_storage_); | 376 parsed_ = StringPiece(parsed_storage_); |
383 } | 377 } |
384 // Clear the quote char so next time we try to parse a string we'll | 378 // Clear the quote char so next time we try to parse a string we'll |
385 // start fresh. | 379 // start fresh. |
386 string_open_ = 0; | 380 string_open_ = 0; |
387 Advance(); | 381 Advance(); |
388 return util::Status::OK; | 382 return util::Status::OK; |
389 } | 383 } |
390 // Normal character, just advance past it. | 384 // Normal character, just advance past it. |
(...skipping 79 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
470 return util::Status::OK; | 464 return util::Status::OK; |
471 } | 465 } |
472 | 466 |
473 util::Status JsonStreamParser::ParseNumber() { | 467 util::Status JsonStreamParser::ParseNumber() { |
474 NumberResult number; | 468 NumberResult number; |
475 util::Status result = ParseNumberHelper(&number); | 469 util::Status result = ParseNumberHelper(&number); |
476 if (result.ok()) { | 470 if (result.ok()) { |
477 switch (number.type) { | 471 switch (number.type) { |
478 case NumberResult::DOUBLE: | 472 case NumberResult::DOUBLE: |
479 ow_->RenderDouble(key_, number.double_val); | 473 ow_->RenderDouble(key_, number.double_val); |
480 key_ = StringPiece(); | 474 key_.clear(); |
481 break; | 475 break; |
482 | 476 |
483 case NumberResult::INT: | 477 case NumberResult::INT: |
484 ow_->RenderInt64(key_, number.int_val); | 478 ow_->RenderInt64(key_, number.int_val); |
485 key_ = StringPiece(); | 479 key_.clear(); |
486 break; | 480 break; |
487 | 481 |
488 case NumberResult::UINT: | 482 case NumberResult::UINT: |
489 ow_->RenderUint64(key_, number.uint_val); | 483 ow_->RenderUint64(key_, number.uint_val); |
490 key_ = StringPiece(); | 484 key_.clear(); |
491 break; | 485 break; |
492 | 486 |
493 default: | 487 default: |
494 return ReportFailure("Unable to parse number."); | 488 return ReportFailure("Unable to parse number."); |
495 } | 489 } |
496 } | 490 } |
497 return result; | 491 return result; |
498 } | 492 } |
499 | 493 |
500 util::Status JsonStreamParser::ParseNumberHelper(NumberResult* result) { | 494 util::Status JsonStreamParser::ParseNumberHelper(NumberResult* result) { |
(...skipping 27 matching lines...) Expand all Loading... |
528 } | 522 } |
529 | 523 |
530 // Create a string containing just the number, so we can use safe_strtoX | 524 // Create a string containing just the number, so we can use safe_strtoX |
531 string number = p_.substr(0, index).ToString(); | 525 string number = p_.substr(0, index).ToString(); |
532 | 526 |
533 // Floating point number, parse as a double. | 527 // Floating point number, parse as a double. |
534 if (floating) { | 528 if (floating) { |
535 if (!safe_strtod(number, &result->double_val)) { | 529 if (!safe_strtod(number, &result->double_val)) { |
536 return ReportFailure("Unable to parse number."); | 530 return ReportFailure("Unable to parse number."); |
537 } | 531 } |
538 if (!loose_float_number_conversion_ && | |
539 !MathLimits<double>::IsFinite(result->double_val)) { | |
540 return ReportFailure("Number exceeds the range of double."); | |
541 } | |
542 result->type = NumberResult::DOUBLE; | 532 result->type = NumberResult::DOUBLE; |
543 p_.remove_prefix(index); | 533 p_.remove_prefix(index); |
544 return util::Status::OK; | 534 return util::Status::OK; |
545 } | 535 } |
546 | 536 |
547 // Positive non-floating point number, parse as a uint64. | 537 // Positive non-floating point number, parse as a uint64. |
548 if (!negative) { | 538 if (!negative) { |
549 // Octal/Hex numbers are not valid JSON values. | 539 // Octal/Hex numbers are not valid JSON values. |
550 if (number.length() >= 2 && number[0] == '0') { | 540 if (number.length() >= 2 && number[0] == '0') { |
551 return ReportFailure("Octal/hex numbers are not valid JSON values."); | 541 return ReportFailure("Octal/hex numbers are not valid JSON values."); |
(...skipping 16 matching lines...) Expand all Loading... |
568 } | 558 } |
569 result->type = NumberResult::INT; | 559 result->type = NumberResult::INT; |
570 p_.remove_prefix(index); | 560 p_.remove_prefix(index); |
571 return util::Status::OK; | 561 return util::Status::OK; |
572 } | 562 } |
573 | 563 |
574 util::Status JsonStreamParser::HandleBeginObject() { | 564 util::Status JsonStreamParser::HandleBeginObject() { |
575 GOOGLE_DCHECK_EQ('{', *p_.data()); | 565 GOOGLE_DCHECK_EQ('{', *p_.data()); |
576 Advance(); | 566 Advance(); |
577 ow_->StartObject(key_); | 567 ow_->StartObject(key_); |
578 key_ = StringPiece(); | 568 key_.clear(); |
579 stack_.push(ENTRY); | 569 stack_.push(ENTRY); |
580 return util::Status::OK; | 570 return util::Status::OK; |
581 } | 571 } |
582 | 572 |
583 util::Status JsonStreamParser::ParseObjectMid(TokenType type) { | 573 util::Status JsonStreamParser::ParseObjectMid(TokenType type) { |
584 if (type == UNKNOWN) { | 574 if (type == UNKNOWN) { |
585 return ReportUnknown("Expected , or } after key:value pair."); | 575 return ReportUnknown("Expected , or } after key:value pair."); |
586 } | 576 } |
587 | 577 |
588 // Object is complete, advance past the comma and render the EndObject. | 578 // Object is complete, advance past the comma and render the EndObject. |
(...skipping 29 matching lines...) Expand all Loading... |
618 // Key is a string (standard JSON), parse it and store the string. | 608 // Key is a string (standard JSON), parse it and store the string. |
619 result = ParseStringHelper(); | 609 result = ParseStringHelper(); |
620 if (result.ok()) { | 610 if (result.ok()) { |
621 key_storage_.clear(); | 611 key_storage_.clear(); |
622 if (!parsed_storage_.empty()) { | 612 if (!parsed_storage_.empty()) { |
623 parsed_storage_.swap(key_storage_); | 613 parsed_storage_.swap(key_storage_); |
624 key_ = StringPiece(key_storage_); | 614 key_ = StringPiece(key_storage_); |
625 } else { | 615 } else { |
626 key_ = parsed_; | 616 key_ = parsed_; |
627 } | 617 } |
628 parsed_ = StringPiece(); | 618 parsed_.clear(); |
629 } | 619 } |
630 } else if (type == BEGIN_KEY) { | 620 } else if (type == BEGIN_KEY) { |
631 // Key is a bare key (back compat), create a StringPiece pointing to it. | 621 // Key is a bare key (back compat), create a StringPiece pointing to it. |
632 result = ParseKey(); | 622 result = ParseKey(); |
633 } else { | 623 } else { |
634 // Unknown key type, report an error. | 624 // Unknown key type, report an error. |
635 result = ReportFailure("Expected an object key or }."); | 625 result = ReportFailure("Expected an object key or }."); |
636 } | 626 } |
637 // On success we next expect an entry mid ':' then an object mid ',' or '}' | 627 // On success we next expect an entry mid ':' then an object mid ',' or '}' |
638 if (result.ok()) { | 628 if (result.ok()) { |
(...skipping 12 matching lines...) Expand all Loading... |
651 stack_.push(VALUE); | 641 stack_.push(VALUE); |
652 return util::Status::OK; | 642 return util::Status::OK; |
653 } | 643 } |
654 return ReportFailure("Expected : between key:value pair."); | 644 return ReportFailure("Expected : between key:value pair."); |
655 } | 645 } |
656 | 646 |
657 util::Status JsonStreamParser::HandleBeginArray() { | 647 util::Status JsonStreamParser::HandleBeginArray() { |
658 GOOGLE_DCHECK_EQ('[', *p_.data()); | 648 GOOGLE_DCHECK_EQ('[', *p_.data()); |
659 Advance(); | 649 Advance(); |
660 ow_->StartList(key_); | 650 ow_->StartList(key_); |
661 key_ = StringPiece(); | 651 key_.clear(); |
662 stack_.push(ARRAY_VALUE); | 652 stack_.push(ARRAY_VALUE); |
663 return util::Status::OK; | 653 return util::Status::OK; |
664 } | 654 } |
665 | 655 |
666 util::Status JsonStreamParser::ParseArrayValue(TokenType type) { | 656 util::Status JsonStreamParser::ParseArrayValue(TokenType type) { |
667 if (type == UNKNOWN) { | 657 if (type == UNKNOWN) { |
668 return ReportUnknown("Expected a value or ] within an array."); | 658 return ReportUnknown("Expected a value or ] within an array."); |
669 } | 659 } |
670 | 660 |
671 if (type == END_ARRAY) { | 661 if (type == END_ARRAY) { |
672 ow_->EndList(); | 662 ow_->EndList(); |
673 Advance(); | 663 Advance(); |
674 return util::Status::OK; | 664 return util::Status::OK; |
675 } | 665 } |
676 | 666 |
677 // The ParseValue call may push something onto the stack so we need to make | 667 // The ParseValue call may push something onto the stack so we need to make |
678 // sure an ARRAY_MID is after it, so we push it on now. Also, the parsing of | 668 // sure an ARRAY_MID is after it, so we push it on now. |
679 // empty-null array value is relying on this ARRAY_MID token. | |
680 stack_.push(ARRAY_MID); | 669 stack_.push(ARRAY_MID); |
681 util::Status result = ParseValue(type); | 670 util::Status result = ParseValue(type); |
682 if (result == util::Status::CANCELLED) { | 671 if (result == util::Status::CANCELLED) { |
683 // If we were cancelled, pop back off the ARRAY_MID so we don't try to | 672 // If we were cancelled, pop back off the ARRAY_MID so we don't try to |
684 // push it on again when we try over. | 673 // push it on again when we try over. |
685 stack_.pop(); | 674 stack_.pop(); |
686 } | 675 } |
687 return result; | 676 return result; |
688 } | 677 } |
689 | 678 |
(...skipping 13 matching lines...) Expand all Loading... |
703 Advance(); | 692 Advance(); |
704 stack_.push(ARRAY_VALUE); | 693 stack_.push(ARRAY_VALUE); |
705 return util::Status::OK; | 694 return util::Status::OK; |
706 } | 695 } |
707 // Illegal token after array value. | 696 // Illegal token after array value. |
708 return ReportFailure("Expected , or ] after array value."); | 697 return ReportFailure("Expected , or ] after array value."); |
709 } | 698 } |
710 | 699 |
711 util::Status JsonStreamParser::ParseTrue() { | 700 util::Status JsonStreamParser::ParseTrue() { |
712 ow_->RenderBool(key_, true); | 701 ow_->RenderBool(key_, true); |
713 key_ = StringPiece(); | 702 key_.clear(); |
714 p_.remove_prefix(true_len); | 703 p_.remove_prefix(true_len); |
715 return util::Status::OK; | 704 return util::Status::OK; |
716 } | 705 } |
717 | 706 |
718 util::Status JsonStreamParser::ParseFalse() { | 707 util::Status JsonStreamParser::ParseFalse() { |
719 ow_->RenderBool(key_, false); | 708 ow_->RenderBool(key_, false); |
720 key_ = StringPiece(); | 709 key_.clear(); |
721 p_.remove_prefix(false_len); | 710 p_.remove_prefix(false_len); |
722 return util::Status::OK; | 711 return util::Status::OK; |
723 } | 712 } |
724 | 713 |
725 util::Status JsonStreamParser::ParseNull() { | 714 util::Status JsonStreamParser::ParseNull() { |
726 ow_->RenderNull(key_); | 715 ow_->RenderNull(key_); |
727 key_ = StringPiece(); | 716 key_.clear(); |
728 p_.remove_prefix(null_len); | 717 p_.remove_prefix(null_len); |
729 return util::Status::OK; | 718 return util::Status::OK; |
730 } | 719 } |
731 | 720 |
732 util::Status JsonStreamParser::ParseEmptyNull() { | |
733 ow_->RenderNull(key_); | |
734 key_ = StringPiece(); | |
735 return util::Status::OK; | |
736 } | |
737 | |
738 bool JsonStreamParser::IsEmptyNullAllowed(TokenType type) { | |
739 if (stack_.empty()) return false; | |
740 return (stack_.top() == ARRAY_MID && type == VALUE_SEPARATOR) || | |
741 stack_.top() == OBJ_MID; | |
742 } | |
743 | |
744 util::Status JsonStreamParser::ReportFailure(StringPiece message) { | 721 util::Status JsonStreamParser::ReportFailure(StringPiece message) { |
745 static const int kContextLength = 20; | 722 static const int kContextLength = 20; |
746 const char* p_start = p_.data(); | 723 const char* p_start = p_.data(); |
747 const char* json_start = json_.data(); | 724 const char* json_start = json_.data(); |
748 const char* begin = std::max(p_start - kContextLength, json_start); | 725 const char* begin = std::max(p_start - kContextLength, json_start); |
749 const char* end = | 726 const char* end = |
750 std::min(p_start + kContextLength, json_start + json_.size()); | 727 std::min(p_start + kContextLength, json_start + json_.size()); |
751 StringPiece segment(begin, end - begin); | 728 StringPiece segment(begin, end - begin); |
752 string location(p_start - begin, ' '); | 729 string location(p_start - begin, ' '); |
753 location.push_back('^'); | 730 location.push_back('^'); |
(...skipping 80 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
834 // can't parse what we have so far. So we don't report an error and just | 811 // can't parse what we have so far. So we don't report an error and just |
835 // return UNKNOWN so we can try again later when we have more data, or if we | 812 // return UNKNOWN so we can try again later when we have more data, or if we |
836 // finish and we have leftovers. | 813 // finish and we have leftovers. |
837 return UNKNOWN; | 814 return UNKNOWN; |
838 } | 815 } |
839 | 816 |
840 } // namespace converter | 817 } // namespace converter |
841 } // namespace util | 818 } // namespace util |
842 } // namespace protobuf | 819 } // namespace protobuf |
843 } // namespace google | 820 } // namespace google |
OLD | NEW |