rapidyaml  0.16.0
parseandemitYAML,anddoitfast
parse_engine.def.hpp
Go to the documentation of this file.
1 #ifndef C4_YML_PARSE_ENGINE_DEF_HPP_
2 #define C4_YML_PARSE_ENGINE_DEF_HPP_
3 
4 #ifndef C4_YML_PARSE_ENGINE_HPP_
5 #include "c4/yml/parse_engine.hpp"
6 #endif
7 #ifndef C4_CHARCONV_HPP_
8 #include "c4/charconv.hpp"
9 #endif
10 #ifndef C4_UTF_HPP_
11 #include "c4/utf.hpp"
12 #endif
13 #ifndef C4_YML_FILTER_PROCESSOR_HPP_
14 #include "c4/yml/filter_processor.hpp"
15 #endif
16 #ifndef C4_YML_TAG_HPP_
17 #include "c4/yml/tag.hpp"
18 #endif
19 #ifndef C4_YML_NODE_TYPE_HPP_
20 #include "c4/yml/node_type.hpp"
21 #endif
22 
23 #ifndef C4_YML_DETAIL_DBGPRINT_HPP_
24 #include "c4/yml/detail/dbgprint.hpp"
25 #endif
26 
27 #ifdef RYML_DBG
28 #ifndef C4_DUMP_HPP_
29 #include <c4/dump.hpp>
30 #endif
31 #define _c4err(...) \
32  do { RYML_DEBUG_BREAK(); this->_err(RYML_LOC_HERE(), __VA_ARGS__); } while(0)
33 #else
34 #define _c4err(...) \
35  this->_err(RYML_LOC_HERE(), __VA_ARGS__)
36 #endif
37 #define _c4assert(...) \
38  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, __VA_ARGS__, m_evt_handler->m_curr->pos)
39 
40 
41 #if defined(RYML_WITH_TAB_TOKENS)
42 #define RYML_WITH_TAB_TOKENS_(...) __VA_ARGS__
43 #define RYML_WITHOUT_TAB_TOKENS_(...)
44 #define RYML_WITH_OR_WITHOUT_TAB_TOKENS_(with, without) with
45 #else
46 #define RYML_WITH_TAB_TOKENS_(...)
47 #define RYML_WITHOUT_TAB_TOKENS_(...) __VA_ARGS__
48 #define RYML_WITH_OR_WITHOUT_TAB_TOKENS_(with, without) without
49 #endif
50 
51 
52 // scaffold:
53 #define _c4dbgnextline() \
54  do { \
55  _c4dbgq("\n-----------"); \
56  _c4dbgt("handling line={}, offset={}B", \
57  m_evt_handler->m_curr->pos.line, \
58  m_evt_handler->m_curr->pos.offset); \
59  } while(0)
60 
61 
62 C4_SUPPRESS_WARNING_MSVC_PUSH
63 C4_SUPPRESS_WARNING_MSVC(4296) // expression is always 'boolean_value'
64 C4_SUPPRESS_WARNING_MSVC(4702) // unreachable code
65 C4_SUPPRESS_WARNING_GCC_CLANG_PUSH
66 C4_SUPPRESS_WARNING_GCC_CLANG("-Wtype-limits") // to remove a warning on an assertion that a size_t >= 0. Later on, this size_t will turn into a template argument, and then it can become < 0.
67 C4_SUPPRESS_WARNING_GCC_CLANG("-Wformat-nonliteral")
68 C4_SUPPRESS_WARNING_GCC_CLANG("-Wold-style-cast")
69 #if defined(__GNUC__) && (__GNUC__ >= 6)
70 C4_SUPPRESS_WARNING_GCC("-Wnull-dereference")
71 #endif
72 #if defined(__GNUC__) && (__GNUC__ >= 7)
73 C4_SUPPRESS_WARNING_GCC("-Wduplicated-branches")
74 #endif
75 
76 // NOLINTBEGIN(hicpp-signed-bitwise,cppcoreguidelines-avoid-goto,hicpp-avoid-goto,hicpp-multiway-paths-covered,modernize-avoid-c-style-cast)
77 
78 namespace c4 {
79 namespace yml {
80 
81 namespace { // NOLINT
82 
83 C4_HOT C4_ALWAYS_INLINE void _set_first(substr &C4_RESTRICT subject, size_t pos) noexcept
84 {
85  // avoids reassigning the ptr in substr
86  subject.len = pos != npos ? pos : subject.len;
87 }
88 C4_HOT C4_ALWAYS_INLINE void _set_first(csubstr &C4_RESTRICT subject, size_t pos) noexcept
89 {
90  // avoids reassigning the ptr in substr
91  subject.len = pos != npos ? pos : subject.len;
92 }
93 C4_HOT C4_ALWAYS_INLINE void _set_first_strict(substr &C4_RESTRICT subject, size_t pos) RYML_NOEXCEPT
94 {
95  // avoids reassigning the ptr in substr
96  RYML_ASSERT_BASIC_(pos != npos); // LCOV_EXCL_LINE
97  subject.len = pos;
98 }
99 C4_HOT C4_ALWAYS_INLINE void _set_first_strict(csubstr &C4_RESTRICT subject, size_t pos) RYML_NOEXCEPT
100 {
101  // avoids reassigning the ptr in substr
102  RYML_ASSERT_BASIC_(pos != npos); // LCOV_EXCL_LINE
103  subject.len = pos;
104 }
105 
106 C4_HOT C4_ALWAYS_INLINE bool _is_blck_token(csubstr s) RYML_NOEXCEPT
107 {
108  RYML_ASSERT_BASIC_(s.len > 0);
109  RYML_ASSERT_BASIC_(s.str[0] == '-' || s.str[0] == ':' || s.str[0] == '?');
110  return ((s.len == 1) || ((s.str[1] == ' ') RYML_WITH_TAB_TOKENS_( || (s.str[1] == '\t'))));
111 }
112 
113 C4_HOT C4_ALWAYS_INLINE bool _is_blck_seq_token_maybe(csubstr const& C4_RESTRICT s) noexcept
114 {
115  return ((s.len >= 1) && (s.str[0] == '-') && ((s.len == 1) || ((s.str[1] == ' ') RYML_WITH_TAB_TOKENS_( || (s.str[1] == '\t')))));
116 }
117 
118 inline bool _is_doc_begin_token(csubstr s) RYML_NOEXCEPT
119 {
120  RYML_ASSERT_BASIC_(s.begins_with('-'));
121  RYML_ASSERT_BASIC_(!s.ends_with("\n"));
122  RYML_ASSERT_BASIC_(!s.ends_with("\r"));
123  return (s.len >= 3 && s.str[1] == '-' && s.str[2] == '-')
124  && (s.len == 3 || (s.str[3] == ' ' RYML_WITH_TAB_TOKENS_(|| s.str[3] == '\t')));
125 }
126 
127 inline bool _is_doc_end_token(csubstr s) RYML_NOEXCEPT
128 {
129  RYML_ASSERT_BASIC_(s.begins_with('.'));
130  RYML_ASSERT_BASIC_(!s.ends_with("\n"));
131  RYML_ASSERT_BASIC_(!s.ends_with("\r"));
132  return (s.len >= 3 && s.str[1] == '.' && s.str[2] == '.')
133  && (s.len == 3 || (s.str[3] == ' ' RYML_WITH_TAB_TOKENS_(|| s.str[3] == '\t')));
134 }
135 
136 inline bool _is_doc_token(csubstr s) noexcept
137 {
138  if(s.len >= 3)
139  {
140  switch(s.str[0])
141  {
142  case '-':
143  //return _is_doc_begin_token(s); // this was failing with gcc -O2
144  return (s.str[1] == '-' && s.str[2] == '-')
145  && (s.len == 3 || (s.str[3] == ' ' RYML_WITH_TAB_TOKENS_(|| s.str[3] == '\t')));
146  case '.':
147  //return _is_doc_end_token(s); // this was failing with gcc -O2
148  return (s.str[1] == '.' && s.str[2] == '.')
149  && (s.len == 3 || (s.str[3] == ' ' RYML_WITH_TAB_TOKENS_(|| s.str[3] == '\t')));
150  }
151  }
152  return false;
153 }
154 
155 inline size_t _begins_with_special_json_scalar(csubstr s) RYML_NOEXCEPT
156 {
157  RYML_ASSERT_BASIC_(s.len);
158  switch(s.str[0])
159  {
160  case 'f':
161  return s.begins_with("false") ? 5u : 0u;
162  case 't':
163  return s.begins_with("true") ? 4u : 0u;
164  case 'n':
165  return s.begins_with("null") ? 4u : 0u;
166  }
167  return 0u;
168 }
169 
170 
171 //-----------------------------------------------------------------------------
172 
173 C4_ALWAYS_INLINE size_t _extend_from_combined_newline(char nl, char following)
174 {
175  return (nl == '\n' && following == '\r') || (nl == '\r' && following == '\n');
176 }
177 
178 //! look for the next newline chars, and jump to the right of those
179 inline substr _from_next_line(substr rem)
180 {
181  size_t nlpos = rem.first_of("\r\n");
182  if(nlpos == csubstr::npos)
183  return {};
184  const char nl = rem[nlpos];
185  rem = rem.right_of(nlpos);
186  if(rem.empty())
187  return {};
188  if(_extend_from_combined_newline(nl, rem.front()))
189  rem = rem.sub(1);
190  return rem;
191 }
192 
193 
194 //-----------------------------------------------------------------------------
195 
196 inline size_t _count_following_newlines(csubstr r, size_t *C4_RESTRICT i)
197 {
198  RYML_ASSERT_BASIC_(r[*i] == '\n');
199  size_t numnl_following = 0;
200  ++(*i);
201  for( ; *i < r.len; ++(*i))
202  {
203  if(r.str[*i] == '\n')
204  ++numnl_following;
205  // skip leading whitespace
206  else if(r.str[*i] == ' ' || r.str[*i] == '\t' || r.str[*i] == '\r')
207  ;
208  else
209  break;
210  }
211  return numnl_following;
212 }
213 
214 /** @p i is set to the first non whitespace character after the line
215  * @return the number of empty lines after the initial position */
216 inline size_t _count_following_newlines(csubstr r, size_t *C4_RESTRICT i, size_t indentation)
217 {
218  RYML_ASSERT_BASIC_(r[*i] == '\n');
219  size_t numnl_following = 0;
220  ++(*i);
221  if(indentation == 0)
222  {
223  for( ; *i < r.len; ++(*i))
224  {
225  const char c = r.str[*i];
226  if(c == '\n')
227  ++numnl_following;
228  // skip leading whitespace
229  else if(c != ' ' && c != '\t' && c != '\r')
230  break;
231  }
232  }
233  else
234  {
235  for( ; *i < r.len; ++(*i))
236  {
237  char c = r.str[*i];
238  if(c == '\n')
239  {
240  ++numnl_following;
241  // skip the indentation after the newline
242  size_t stop = *i + indentation;
243  for( ; *i < r.len; ++(*i))
244  {
245  c = r.str[*i];
246  if(c != ' ' && c != '\r')
247  break;
248  RYML_ASSERT_BASIC_(*i < stop); // LCOV_EXCL_LINE
249  }
250  C4_UNUSED(stop);
251  }
252  // skip leading whitespace
253  else if(c != ' ' && c != '\t' && c != '\r')
254  {
255  break;
256  }
257  }
258  }
259  return numnl_following;
260 }
261 
262 } // anon namespace
263 
264 
265 //-----------------------------------------------------------------------------
266 //-----------------------------------------------------------------------------
267 //-----------------------------------------------------------------------------
268 
269 template<class EventHandler>
270 ParseEngine<EventHandler>::~ParseEngine() noexcept
271 {
272  _free();
273  _clr();
274 }
275 
276 template<class EventHandler>
277 ParseEngine<EventHandler>::ParseEngine(EventHandler *evt_handler, ParserOptions const& opts)
278  : m_options(opts)
279  , m_evt_handler(evt_handler)
280  , m_pending_anchors()
281  , m_pending_tags()
282  , m_has_directives_yaml(false)
283  , m_has_directives(false)
284  , m_doc_empty(true)
285  , m_prev_colon(npos)
286  , m_prev_val_end(npos)
287  , m_encoding(NOBOM)
288  , m_newline_offsets()
289  , m_newline_offsets_size(0)
290  , m_newline_offsets_capacity(0)
291 {
292  RYML_CHECK_BASIC_(evt_handler);
293 }
294 
295 template<class EventHandler>
296 ParseEngine<EventHandler>::ParseEngine(ParseEngine &&that) noexcept
297  : m_options(that.m_options)
298  , m_evt_handler(that.m_evt_handler)
299  , m_pending_anchors(that.m_pending_anchors)
300  , m_pending_tags(that.m_pending_tags)
301  , m_has_directives_yaml(that.m_has_directives_yaml)
302  , m_has_directives(that.m_has_directives)
303  , m_doc_empty(that.m_doc_empty)
304  , m_prev_colon(npos)
305  , m_prev_val_end(npos)
306  , m_encoding(NOBOM)
307  , m_newline_offsets(that.m_newline_offsets)
308  , m_newline_offsets_size(that.m_newline_offsets_size)
309  , m_newline_offsets_capacity(that.m_newline_offsets_capacity)
310 {
311  that._clr();
312 }
313 
314 template<class EventHandler>
315 ParseEngine<EventHandler>::ParseEngine(ParseEngine const& that)
316  : m_options(that.m_options)
317  , m_evt_handler(that.m_evt_handler)
318  , m_pending_anchors(that.m_pending_anchors)
319  , m_pending_tags(that.m_pending_tags)
320  , m_has_directives_yaml(that.m_has_directives_yaml)
321  , m_has_directives(that.m_has_directives)
322  , m_doc_empty(that.m_doc_empty)
323  , m_prev_colon(npos)
324  , m_prev_val_end(npos)
325  , m_encoding(NOBOM)
326  , m_newline_offsets()
327  , m_newline_offsets_size()
328  , m_newline_offsets_capacity()
329 {
330  if(that.m_newline_offsets_capacity)
331  {
332  _resize_locations(that.m_newline_offsets_capacity);
333  RYML_CHECK_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, m_newline_offsets_capacity == that.m_newline_offsets_capacity);
334  memcpy(m_newline_offsets, that.m_newline_offsets, that.m_newline_offsets_size * sizeof(size_t));
335  m_newline_offsets_size = that.m_newline_offsets_size;
336  }
337 }
338 
339 template<class EventHandler>
340 ParseEngine<EventHandler>& ParseEngine<EventHandler>::operator=(ParseEngine &&that) noexcept
341 {
342  _free();
343  m_options = (that.m_options);
344  m_evt_handler = that.m_evt_handler;
345  m_pending_anchors = that.m_pending_anchors;
346  m_pending_tags = that.m_pending_tags;
347  m_has_directives_yaml = that.m_has_directives_yaml;
348  m_has_directives = that.m_has_directives;
349  m_doc_empty = that.m_doc_empty;
350  m_prev_colon = that.m_prev_colon;
351  m_prev_val_end = that.m_prev_val_end;
352  m_encoding = that.m_encoding;
353  m_newline_offsets = (that.m_newline_offsets);
354  m_newline_offsets_size = (that.m_newline_offsets_size);
355  m_newline_offsets_capacity = (that.m_newline_offsets_capacity);
356  that._clr();
357  return *this;
358 }
359 
360 template<class EventHandler>
361 ParseEngine<EventHandler>& ParseEngine<EventHandler>::operator=(ParseEngine const& that)
362 {
363  if(&that != this)
364  {
365  _free();
366  m_options = (that.m_options);
367  m_evt_handler = that.m_evt_handler;
368  m_pending_anchors = that.m_pending_anchors;
369  m_pending_tags = that.m_pending_tags;
370  m_has_directives_yaml = that.m_has_directives_yaml;
371  m_has_directives = that.m_has_directives;
372  m_doc_empty = that.m_doc_empty;
373  m_prev_colon = that.m_prev_colon;
374  m_prev_val_end = that.m_prev_val_end;
375  m_encoding = that.m_encoding;
376  if(that.m_newline_offsets_capacity > m_newline_offsets_capacity)
377  _resize_locations(that.m_newline_offsets_capacity);
378  RYML_CHECK_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, m_newline_offsets_capacity >= that.m_newline_offsets_capacity);
379  RYML_CHECK_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, m_newline_offsets_capacity >= that.m_newline_offsets_size);
380  memcpy(m_newline_offsets, that.m_newline_offsets, that.m_newline_offsets_size * sizeof(size_t));
381  m_newline_offsets_size = that.m_newline_offsets_size;
382  }
383  return *this;
384 }
385 
386 template<class EventHandler>
387 void ParseEngine<EventHandler>::_clr()
388 {
389  m_options = {};
390  m_evt_handler = {};
391  m_pending_anchors = {};
392  m_pending_tags = {};
393  m_has_directives_yaml = false;
394  m_has_directives = false;
395  m_doc_empty = true;
396  m_prev_colon = npos;
397  m_prev_val_end = npos;
398  m_encoding = NOBOM;
399  m_newline_offsets = {};
400  m_newline_offsets_size = {};
401  m_newline_offsets_capacity = {};
402 }
403 
404 template<class EventHandler>
405 void ParseEngine<EventHandler>::_free()
406 {
407  if(m_newline_offsets)
408  {
409  RYML_CB_FREE_(m_evt_handler->m_stack.m_callbacks, m_newline_offsets, size_t, m_newline_offsets_capacity);
410  m_newline_offsets = nullptr;
411  m_newline_offsets_size = 0u;
412  m_newline_offsets_capacity = 0u;
413  }
414 }
415 
416 
417 //-----------------------------------------------------------------------------
418 
419 template<class EventHandler>
420 void ParseEngine<EventHandler>::_reset()
421 {
422  m_pending_anchors = {};
423  m_pending_tags = {};
424  m_has_directives_yaml = false;
425  m_has_directives = false;
426  m_doc_empty = true;
427  m_prev_colon = npos;
428  m_prev_val_end = npos;
429  m_bom_len = 0;
430  m_encoding = NOBOM;
431  m_bom_line = 0;
432  if(m_options.locations())
433  {
434  _prepare_locations();
435  }
436 }
437 
438 
439 //-----------------------------------------------------------------------------
440 
441 template<class EventHandler>
442 void ParseEngine<EventHandler>::_relocate_arena(csubstr prev_arena, substr next_arena, substr *other)
443 {
444  _c4dbgp("relocate to new arena");
445  const char *pb = prev_arena.str;
446  const char *pe = prev_arena.str + prev_arena.len;
447  #define _ryml_relocate(s) \
448  if((s).str >= pb && (s).str <= pe) \
449  { \
450  (s).str = next_arena.str + ((s).str - pb); \
451  } \
452  ((void)0)
453  for(ParserState &st : m_evt_handler->m_stack)
454  {
455  _ryml_relocate(st.line_contents.rem);
456  _ryml_relocate(st.line_contents.full);
457  }
458  _ryml_relocate(m_evt_handler->m_src);
459  for(size_t i = 0; i < m_pending_tags.num_entries; ++i)
460  {
461  _ryml_relocate(m_pending_tags.annotations[i].str); // LCOV_EXCL_LINE
462  _ryml_relocate(m_pending_tags.annotations[i].orig); // LCOV_EXCL_LINE
463  }
464  for(size_t i = 0; i < m_pending_anchors.num_entries; ++i)
465  {
466  _ryml_relocate(m_pending_anchors.annotations[i].str);
467  _ryml_relocate(m_pending_anchors.annotations[i].orig);
468  }
469  {
470  TagDirectives &tds = m_evt_handler->tag_directives();
471  for(size_t i = 0, sz = tds.size(); i < sz; ++i)
472  {
473  _ryml_relocate(tds.m_directives[i].handle);
474  _ryml_relocate(tds.m_directives[i].prefix);
475  }
476  }
477  {
478  TagCache &tch = m_evt_handler->tag_cache();
479  for(id_type i = 0, sz = tch.m_entries.size(); i < sz; ++i)
480  {
481  _ryml_relocate(tch.m_entries[i].tag);
482  _ryml_relocate(tch.m_entries[i].resolved);
483  }
484  }
485  if(other)
486  {
487  _ryml_relocate(*other);
488  }
489  #undef _ryml_relocate
490 }
491 
492 /** @cond dev */
493 template<class EventHandler>
494 substr ParseEngine<EventHandler>::_alloc_arena(size_t len, substr *other)
495 {
496  csubstr prev = m_evt_handler->arena();
497  substr out = m_evt_handler->alloc_arena(len);
498  substr curr = m_evt_handler->arena();
499  if(curr.str != prev.str)
500  _relocate_arena(prev, curr, other);
501  return out;
502 }
503 /** @endcond */
504 
505 
506 //-----------------------------------------------------------------------------
507 
508 #ifdef RYML_DBG
509 template<class EventHandler>
510 template<class DumpFn>
511 C4_NO_INLINE void ParseEngine<EventHandler>::_fmt_msg(DumpFn &&dumpfn) const
512 {
513  ParserState const *const C4_RESTRICT st = m_evt_handler->m_curr;
514  LineContents const& C4_RESTRICT lc = st->line_contents;
515  csubstr contents = lc.full.first(lc.num_cols);
516  if(contents.len)
517  {
518  // print the yaml src line
519  size_t offs = 3u + to_chars(substr{}, st->pos.line) + to_chars(substr{}, st->pos.col);
520  csubstr m_file = m_evt_handler->m_curr->pos.name;
521  if(m_file.len)
522  {
523  dbg_dump_(std::forward<DumpFn>(dumpfn), "{}:", m_file);
524  offs += m_file.len + 1;
525  }
526  dbg_dump_(std::forward<DumpFn>(dumpfn), "{}:{}: ", st->pos.line, st->pos.col);
527  csubstr maybe_full_content = (contents.len < 80u ? contents : contents.first(80u));
528  csubstr maybe_ellipsis = (contents.len < 80u ? csubstr{} : csubstr("..."));
529  dbg_dump_(std::forward<DumpFn>(dumpfn), "{}{} (size={})\n", escaped_scalar(maybe_full_content, /*escape*/true), maybe_ellipsis, contents.len);
530  // highlight the remaining portion of the previous line
531  size_t firstcol = (size_t)(lc.rem.str - lc.full.str);
532  size_t lastcol = firstcol + lc.rem.len;
533  size_t firstcol_adj = adjust_pos_with_escapes(lc.full, firstcol);
534  size_t len = adjust_pos_with_escapes(lc.rem, lc.rem.len);
535  for(size_t i = 0; i < offs + firstcol_adj; ++i)
536  std::forward<DumpFn>(dumpfn)(" ");
537  std::forward<DumpFn>(dumpfn)("^");
538  for(size_t i = 1, e = (len < 80u ? len : 80u); i < e; ++i)
539  std::forward<DumpFn>(dumpfn)("~");
540  dbg_dump_(std::forward<DumpFn>(dumpfn), "{} (cols {}-{})\n", maybe_ellipsis, firstcol+1, lastcol+1);
541  }
542  else
543  {
544  std::forward<DumpFn>(dumpfn)("\n");
545  }
546  // next line: print the state flags
547  {
548  char flagbuf_[128];
549  dbg_dump_(std::forward<DumpFn>(dumpfn), "top state: {}\n", detail::_parser_flags_to_str(flagbuf_, m_evt_handler->m_curr->flags));
550  }
551 }
552 
553 template<class EventHandler>
554 void ParseEngine<EventHandler>::_print_state_stack(substr buf) const
555 {
556  if(dbg_enabled_())
557  {
558  for(ParserState const& s : m_evt_handler->m_stack)
559  dbg_printf_("state[{}]: ind={} node={} flags={}\n", s.level, s.indref, s.node_id, detail::_parser_flags_to_str(buf, s.flags));
560  }
561 }
562 
563 template<class EventHandler>
564 void ParseEngine<EventHandler>::_print_state_stack() const
565 {
566  char buf[128];
567  _print_state_stack(buf);
568 }
569 #endif
570 
571 
572 //-----------------------------------------------------------------------------
573 
574 template<class EventHandler>
575 template<class ...Args>
576 C4_NORETURN C4_NO_INLINE void ParseEngine<EventHandler>::_err(Location const& cpploc, Location const& ymlloc, const char* fmt, Args const& ...args) const
577 {
578  m_evt_handler->cancel_parse();
579  err_parse(m_evt_handler->m_stack.m_callbacks, ErrorDataParse{cpploc, ymlloc}, fmt, args...);
580 }
581 
582 template<class EventHandler>
583 template<class ...Args>
584 C4_NORETURN C4_NO_INLINE void ParseEngine<EventHandler>::_err(Location const& cpploc, const char *fmt, Args const& ...args) const
585 {
586  m_evt_handler->cancel_parse();
587  err_parse(m_evt_handler->m_stack.m_callbacks, ErrorDataParse{cpploc, m_evt_handler->m_curr->pos}, fmt, args...);
588 }
589 
590 
591 //-----------------------------------------------------------------------------
592 #ifdef RYML_DBG
593 template<class EventHandler>
594 template<class ...Args>
595 void ParseEngine<EventHandler>::_dbg(csubstr fmt, Args const& ...args) const
596 {
597  if(dbg_enabled_())
598  {
599  dbg_printf_(fmt, args...);
600  dbg_dumper_("\n");
601  _fmt_msg(dbg_dumper_);
602  }
603 }
604 #endif
605 
606 
607 //-----------------------------------------------------------------------------
608 template<class EventHandler>
609 bool ParseEngine<EventHandler>::_finished_file() const
610 {
611  bool ret = m_evt_handler->m_curr->pos.offset >= _buf().len;
612  #ifdef RYML_DBG
613  if(ret)
614  {
615  _c4dbgp("finished file!!!");
616  }
617  #endif
618  return ret;
619 }
620 
621 template<class EventHandler>
622 C4_HOT C4_ALWAYS_INLINE bool ParseEngine<EventHandler>::_finished_line() const // LCOV_EXCL_LINE
623 {
624  return m_evt_handler->m_curr->line_contents.rem.empty();
625 }
626 
627 
628 //-----------------------------------------------------------------------------
629 
630 template<class EventHandler>
631 void ParseEngine<EventHandler>::_maybe_skip_whitespace_tokens()
632 {
633  if(m_evt_handler->m_curr->line_contents.rem.len && (m_evt_handler->m_curr->line_contents.rem.str[0] == ' ' RYML_WITH_TAB_TOKENS_(|| m_evt_handler->m_curr->line_contents.rem.str[0] == '\t')))
634  {
635  size_t pos = m_evt_handler->m_curr->line_contents.rem.first_not_of(RYML_WITH_OR_WITHOUT_TAB_TOKENS_(" \t", ' '));
636  if(pos == npos)
637  pos = m_evt_handler->m_curr->line_contents.rem.len; // maybe the line is just all whitespace
638  _c4dbgpf("skip {} whitespace characters", pos);
639  _line_progressed(pos);
640  }
641 }
642 
643 template<class EventHandler>
644 void ParseEngine<EventHandler>::_maybe_skipchars(char c)
645 {
646  if(m_evt_handler->m_curr->line_contents.rem.len && m_evt_handler->m_curr->line_contents.rem.str[0] == c)
647  {
648  size_t pos = m_evt_handler->m_curr->line_contents.rem.first_not_of(c);
649  if(pos == npos)
650  pos = m_evt_handler->m_curr->line_contents.rem.len; // maybe the line is just all c
651  _c4dbgpf("skip {}x'{}'", pos, _c4prc(c));
652  _line_progressed(pos);
653  }
654 }
655 
656 template<class EventHandler>
657 template<size_t N>
658 void ParseEngine<EventHandler>::_skipchars(const char (&chars)[N])
659 {
660  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->line_contents.rem.begins_with_any(chars), m_evt_handler->m_curr->pos);
661  size_t pos = m_evt_handler->m_curr->line_contents.rem.first_not_of(chars);
662  if(pos == npos)
663  pos = m_evt_handler->m_curr->line_contents.rem.len; // maybe the line is just whitespace
664  _c4dbgpf("skip {} characters", pos);
665  _line_progressed(pos);
666 }
667 
668 template<class EventHandler>
669 void ParseEngine<EventHandler>::_skip_comment()
670 {
671  LineContents const& C4_RESTRICT lc = m_evt_handler->m_curr->line_contents;
672  const size_t col = m_evt_handler->m_curr->pos.col - 1u;
673  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, lc.rem.begins_with('#'), m_evt_handler->m_curr->pos);
674  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, lc.rem.is_sub(lc.full), m_evt_handler->m_curr->pos);
675  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->pos.col >= 1, m_evt_handler->m_curr->pos); // 1-based
676  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, col == ((size_t)(lc.rem.str - lc.full.str)), m_evt_handler->m_curr->pos);
677  // raise an error if the comment is not preceded by whitespace
678  if(lc.rem.str != lc.full.str) // not at line beginning
679  {
680  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, col > 0, m_evt_handler->m_curr->pos);
681  const char prev = lc.full.str[col - 1u];
682  if C4_UNLIKELY(prev != ' ' && prev != '\t')
683  _c4err("comment not preceded by whitespace");
684  }
685  _c4dbgpf("comment was '{}'", m_evt_handler->m_curr->line_contents.rem);
686  _line_progressed(m_evt_handler->m_curr->line_contents.rem.len);
687 }
688 
689 template<class EventHandler>
690 void ParseEngine<EventHandler>::_maybe_skip_comment_strict()
691 {
692  size_t pos = m_evt_handler->m_curr->line_contents.rem.first_not_of(" \t");
693  if(pos != npos)
694  {
695  if('#' == m_evt_handler->m_curr->line_contents.rem[pos])
696  {
697  _line_progressed(pos);
698  _skip_comment();
699  }
700  }
701 }
702 
703 template<class EventHandler>
704 void ParseEngine<EventHandler>::_maybe_skip_comment()
705 {
706  size_t pos = m_evt_handler->m_curr->line_contents.rem.first_not_of(" \t");
707  if(pos != npos)
708  {
709  if('#' == m_evt_handler->m_curr->line_contents.rem[pos])
710  {
711  _line_progressed(pos);
712  _skip_comment();
713  }
714  }
715  else
716  {
717  _line_progressed(m_evt_handler->m_curr->line_contents.rem.len);
718  }
719 }
720 
721 template<class EventHandler>
722 bool ParseEngine<EventHandler>::_maybe_scan_following_colon() noexcept
723 {
724  size_t pos = m_evt_handler->m_curr->line_contents.rem.first_not_of(" \t");
725  if(pos != npos)
726  {
727  if(':' == m_evt_handler->m_curr->line_contents.rem[pos])
728  {
729  // bump pos to skip the colon as well, and check the colon
730  // is followed by space or tab
731  if(++pos < m_evt_handler->m_curr->line_contents.rem.len)
732  {
733  const char next = m_evt_handler->m_curr->line_contents.rem.str[pos];
734  if(next == ' ' RYML_WITH_TAB_TOKENS_(|| next == '\t'))
735  ++pos;
736  else
737  return false;
738  }
739  _line_progressed(pos);
740  return true;
741  }
742  }
743  else
744  {
745  _line_progressed(m_evt_handler->m_curr->line_contents.rem.len);
746  }
747  return false;
748 }
749 
750 
751 //-----------------------------------------------------------------------------
752 
753 template<class EventHandler>
754 csubstr ParseEngine<EventHandler>::_scan_anchor()
755 {
756  csubstr s = m_evt_handler->m_curr->line_contents.rem;
757  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.begins_with('&'), m_evt_handler->m_curr->pos);
758  csubstr anchor = s.range(1, s.first_of(" ,]}\t"));
759  _line_progressed(1u + anchor.len);
760  _maybe_skipchars(' ');
761  return anchor;
762 }
763 
764 template<class EventHandler>
765 csubstr ParseEngine<EventHandler>::_scan_ref_seq()
766 {
767  csubstr s = m_evt_handler->m_curr->line_contents.rem;
768  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.begins_with('*'), m_evt_handler->m_curr->pos);
769  _set_first(s, s.first_of(" ,]\t"));
770  _line_progressed(s.len);
771  return s;
772 }
773 
774 template<class EventHandler>
775 csubstr ParseEngine<EventHandler>::_scan_ref_map()
776 {
777  csubstr s = m_evt_handler->m_curr->line_contents.rem;
778  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.begins_with('*'), m_evt_handler->m_curr->pos);
779  _set_first(s, s.first_of(" ,}\t"));
780  _line_progressed(s.len);
781  return s;
782 }
783 
784 template<class EventHandler>
785 csubstr ParseEngine<EventHandler>::_scan_tag()
786 {
787  csubstr t = m_evt_handler->m_curr->line_contents.rem;
788  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, t.begins_with('!'), m_evt_handler->m_curr->pos);
789  if(!t.begins_with("!<"))
790  {
791  _c4dbgp("begins with '!'");
792  _set_first(t, t.first_of(" ,]}\t"));
793  if C4_UNLIKELY(t.first_of("[{") != npos)
794  _c4err("invalid tag");
795  _line_progressed(t.len);
796  if(m_options.resolve_tags_all() || (m_options.resolve_tags() && is_custom_tag(t)))
797  t = _resolve_tag(t);
798  }
799  else
800  {
801  _c4dbgp("begins with '!<'");
802  size_t pos = t.find('>');
803  if C4_UNLIKELY(pos == npos)
804  _c4err("invalid tag");
805  _set_first_strict(t, pos+1);
806  _line_progressed(t.len);
807  t = t.sub(1);
808  }
809  _maybe_skip_whitespace_tokens();
810  return t;
811 }
812 
813 template<class EventHandler>
814 csubstr ParseEngine<EventHandler>::_scan_tag(csubstr *orig)
815 {
816  csubstr t = m_evt_handler->m_curr->line_contents.rem;
817  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, t.begins_with('!'), m_evt_handler->m_curr->pos);
818  if(!t.begins_with("!<"))
819  {
820  _c4dbgp("begins with '!'");
821  _set_first(t, t.first_of(" ,\t"));
822  if C4_UNLIKELY(t.first_of("[{") != npos)
823  _c4err("invalid tag");
824  _line_progressed(t.len);
825  *orig = t;
826  if(m_options.resolve_tags_all() || (m_options.resolve_tags() && is_custom_tag(t)))
827  t = _resolve_tag(t);
828  }
829  else
830  {
831  _c4dbgp("begins with '!<'");
832  size_t pos = t.find('>');
833  if C4_UNLIKELY(pos == npos)
834  _c4err("invalid tag");
835  _set_first_strict(t, pos+1);
836  _line_progressed(t.len);
837  *orig = t;
838  t = t.sub(1);
839  }
840  _maybe_skip_whitespace_tokens();
841  return t;
842 }
843 
844 
845 //-----------------------------------------------------------------------------
846 
847 template<class EventHandler>
848 bool ParseEngine<EventHandler>::_is_valid_start_scalar_plain_flow_check_block_token(csubstr s)
849 {
850  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.len > 0, m_evt_handler->m_curr->pos);
851  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.begins_with_any(":-"), m_evt_handler->m_curr->pos);
852  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.count('\n') == 0, m_evt_handler->m_curr->pos);
853  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.count('\r') == 0, m_evt_handler->m_curr->pos);
854  if(s.len > 1)
855  {
856  switch(s.str[1])
857  {
858  case ' ':
859  case ',':
860  case '}':
861  case ']':
862  case '\t':
863  if(s.str[0] == ':')
864  {
865  _c4dbgpf("not a scalar: found non-scalar token '{}{}'", s.str[0], s.str[1]);
866  return false;
867  }
868  else
869  {
870  _c4err("invalid scalar");
871  }
872  break;
873  case '{':
874  case '[':
875  _c4err("invalid token \":{}\"", _c4prc(s.str[1]));
876  break;
877  default:
878  break;
879  }
880  }
881  else
882  {
883  if(s.str[0] == '-')
884  _c4err("invalid scalar");
885  return false;
886  }
887  return true;
888 }
889 
890 template<class EventHandler>
891 bool ParseEngine<EventHandler>::_is_valid_start_scalar_plain_flow_check_qmrk(csubstr s)
892 {
893  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.len > 0, m_evt_handler->m_curr->pos);
894  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s[0] == '?', m_evt_handler->m_curr->pos);
895  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.count('\n') == 0, m_evt_handler->m_curr->pos);
896  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.count('\r') == 0, m_evt_handler->m_curr->pos);
897  if(s.len > 1)
898  {
899  switch(s.str[1])
900  {
901  case ' ':
902  case '\t':
903  _c4dbgpf("not a scalar: found non-scalar token '?{}'", _c4prc(s.str[1]));
904  return false;
905  case '{':
906  case '}':
907  case '[':
908  case ']':
909  _c4err("invalid token \"?{}\"", _c4prc(s.str[1]));
910  break;
911  default:
912  break;
913  }
914  }
915  else
916  {
917  return false;
918  }
919  return true;
920 }
921 
922 
923 template<class EventHandler>
924 bool ParseEngine<EventHandler>::_is_valid_start_scalar_plain_flow(csubstr s)
925 {
926  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, !s.empty(), m_evt_handler->m_curr->pos);
927  // it's not a scalar if it starts with any of these characters:
928  switch(s.str[0])
929  {
930  // these are all legal tokens which mean no scalar is starting:
931  case '[':
932  case ']':
933  case '{':
934  case '}':
935  case '&':
936  case '*':
937  case '!':
938  case '|':
939  case '>':
940  case '#':
941  case ',':
942  _c4dbgpf("not a scalar: found non-scalar token '{}'", _c4prc(s.str[0]));
943  return false;
944  // '-' and ':' are illegal at the beginning if not followed by a scalar character
945  case '-':
946  case ':':
947  _c4dbgpf("suspicious token='{}' len={}", _c4prc(s.str[0]), s.len);
948  return _is_valid_start_scalar_plain_flow_check_block_token(s);
949  case '?':
950  _c4dbgpf("qmrk='{}' len={}", _c4prc(s.str[0]), s.len);
951  return _is_valid_start_scalar_plain_flow_check_qmrk(s);
952  // everything else is a legal starting character
953  default:
954  return true;
955  }
956 }
957 
958 
959 template<class EventHandler>
960 bool ParseEngine<EventHandler>::_scan_scalar_plain_handle_newline(csubstr s, size_t offs)
961 {
962  _c4dbgpf("newl[PLAIN]: found '\\n'. offs={} line={} sofar={}", offs, m_evt_handler->m_curr->pos.line, prs_(s.first(offs), true));
963  if(s.len > offs + 1)
964  {
965  _c4dbgp("newl[PLAIN]: buffer continues");
966  csubstr next_line = s.sub(offs + 1);
967  size_t next_line_indentation = next_line.first_not_of(' ');
968  if(next_line_indentation != npos)
969  {
970  _c4dbgpf("newl[PLAIN]: line={} indentation={} indref={}", m_evt_handler->m_curr->pos.line + 1, next_line_indentation, m_evt_handler->m_curr->indref);
971  next_line = next_line.first(next_line.first_of("\n\r"));
972  _c4dbgpf("newl[PLAIN]: has indentation. next_line={}", prs_(next_line));
973  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, next_line_indentation <= next_line.len, m_evt_handler->m_curr->pos);
974  if C4_LIKELY(next_line_indentation >= m_evt_handler->m_curr->indref)
975  {
976  _c4dbgp("newl[PLAIN]: larger indentation");
977  next_line = next_line.sub(next_line_indentation);
978  }
979  else if C4_UNLIKELY(next_line.len && next_line.triml(' ').len)
980  {
981  _c4dbgp("newl[PLAIN]: err, smaller indentation");
982  _line_progressed(m_evt_handler->m_curr->line_contents.rem.len);
983  _line_ended();
984  _scan_line();
985  if(m_evt_handler->m_curr->line_contents.indentation != npos)
986  _line_progressed(m_evt_handler->m_curr->line_contents.indentation);
987  _c4err("parse error"); // cannot reduce indentation here
988  }
989  _c4dbgpf("newl[PLAIN]: next_line.len={}", next_line.len);
990  if(next_line.len)
991  {
992  size_t fno = next_line.first_not_of(" \t");
993  if(fno != csubstr::npos)
994  {
995  _c4assert(fno < next_line.len);
996  switch(next_line.str[fno])
997  {
998  case ',': case ']': case '#':
999  _c4dbgpf("newl[PLAIN]: found terminating character beginning next line: '{}'", next_line.str[fno]);
1000  return false;
1001  case ':': // cannot be succeeded by whitespace
1002  _c4dbgp("newl[PLAIN]: found :");
1003  if(fno + 1 == next_line.len || _is_blck_token(next_line.sub(fno)))
1004  {
1005  _c4dbgpf("newl[PLAIN]: found terminating character beginning next line: '{}'", next_line.str[fno]);
1006  return false;
1007  }
1008  break;
1009  }
1010  }
1011  }
1012  }
1013  }
1014  _line_progressed(m_evt_handler->m_curr->line_contents.rem.len);
1015  _line_ended();
1016  _scan_line();
1017  return true;
1018 }
1019 
1020 template<class EventHandler>
1021 bool ParseEngine<EventHandler>::_scan_scalar_plain_seq_flow(ScannedScalar *C4_RESTRICT sc)
1022 {
1023  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RMAP), m_evt_handler->m_curr->pos);
1024  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RBLCK), m_evt_handler->m_curr->pos);
1025  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RSEQ|RSEQIMAP), m_evt_handler->m_curr->pos);
1026  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RFLOW), m_evt_handler->m_curr->pos);
1027  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RVAL), m_evt_handler->m_curr->pos);
1028 
1029  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, !m_evt_handler->m_curr->line_contents.rem.begins_with(' '), m_evt_handler->m_curr->pos);
1030  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, !m_evt_handler->m_curr->line_contents.rem.begins_with('\n'), m_evt_handler->m_curr->pos);
1031 
1032  if(!m_evt_handler->m_curr->line_contents.rem.len || !_is_valid_start_scalar_plain_flow(m_evt_handler->m_curr->line_contents.rem))
1033  return false;
1034 
1035  substr s = _buf().sub(m_evt_handler->m_curr->pos.offset);
1036  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.begins_with(m_evt_handler->m_curr->line_contents.rem), m_evt_handler->m_curr->pos);
1037 
1038  _c4dbgp("scanning seqflow scalar...");
1039 
1040  bool needs_filter = false;
1041  size_t col = 0; // zero-based column
1042  size_t offs = 0; // offset
1043  for( ; offs < s.len; ++offs, ++col)
1044  {
1045  const char c = s.str[offs];
1046  switch(c)
1047  {
1048  case ',':
1049  case ']':
1050  _c4dbgpf("found terminating character at {}: '{}'", offs, c);
1051  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, offs > 0, m_evt_handler->m_curr->pos);
1052  goto ended_scalar;
1053  case '\n':
1054  _c4dbgpf("found '\\n' at col={}", col);
1055  if(!_scan_scalar_plain_handle_newline(s, offs))
1056  goto ended_scalar;
1057  col = (size_t)-1; // so that col is 0 in the next loop iteration
1058  needs_filter = true;
1059  break;
1060  case '\r':
1061  --col; // don't count \r when calling _line_progressed()
1062  needs_filter = true;
1063  break;
1064  case ':':
1065  _c4dbgp("found suspicious ':'");
1066  if(s.len > offs + 1)
1067  {
1068  char next = s.str[offs + 1];
1069  _c4dbgpf("next char is '{}'", _c4prc(next));
1070  if(next == '\r')
1071  {
1072  csubstr after = s.sub(offs + 1).triml('\r');
1073  if(after.len)
1074  {
1075  next = after.str[0];
1076  _c4dbgpf("skip \\r to '{}'", _c4prc(next));
1077  }
1078  }
1079  // no else here.
1080  if(next == ' ' RYML_WITH_TAB_TOKENS_(|| next == '\t') || next == ',' || next == '\n' || next == ']')
1081  {
1082  _c4dbgp("map starting!");
1083  goto ended_scalar;
1084  }
1085  else
1086  {
1087  _c4dbgp("':' nothing to see here");
1088  }
1089  }
1090  else
1091  {
1092  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.len == offs + 1, m_evt_handler->m_curr->pos);
1093  _line_progressed(col);
1094  _c4err("missing termination: '{}'", c); // noreturn
1095  }
1096  break;
1097  case '#':
1098  {
1099  _c4dbgp("found suspicious '#'");
1100  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, offs > 0, m_evt_handler->m_curr->pos);
1101  char prev = s.str[offs - 1];
1102  if(prev == ' ' RYML_WITH_TAB_TOKENS_(|| prev == '\t'))
1103  {
1104  _c4dbgpf("found terminating character at {}: '{}'", offs, c);
1105  goto ended_scalar;
1106  }
1107  }
1108  break;
1109  case '[':
1110  case '{':
1111  case '}':
1112  _line_progressed(col); // advance to report the proper position in the error
1113  _c4err("invalid character: '{}'", c); // noreturn
1114  case '-':
1115  case '.':
1116  _c4dbgpf("doc token character: '{}', offs={}", c, offs);
1117  if(offs == 0 && m_evt_handler->m_curr->at_line_beginning())
1118  {
1119  _c4dbgp("at line beginning");
1120  if(s.len >= 3 && s.str[1] == c && s.str[2] == c)
1121  {
1122  _c4err("parse error"); // no return
1123  }
1124  }
1125  break;
1126  default:
1127  ;
1128  }
1129  }
1130 
1131 ended_scalar:
1132 
1133  _line_progressed(col);
1134  _set_first(s, offs);
1135  sc->scalar = s.trimr(RYML_WITH_OR_WITHOUT_TAB_TOKENS_(" \t", ' '));
1136  sc->needs_filter = needs_filter;
1137 
1138  _c4prscalar("scanned plain scalar", sc->scalar, /*keep_newlines*/true);
1139 
1140  return true;
1141 }
1142 
1143 template<class EventHandler>
1144 bool ParseEngine<EventHandler>::_scan_scalar_plain_map_flow(ScannedScalar *C4_RESTRICT sc)
1145 {
1146  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RSEQ) || has_any(RSEQIMAP), m_evt_handler->m_curr->pos);
1147  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RBLCK), m_evt_handler->m_curr->pos);
1148  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RMAP|RSEQIMAP), m_evt_handler->m_curr->pos);
1149  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RFLOW), m_evt_handler->m_curr->pos);
1150  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RKEY|RVAL|QMRK), m_evt_handler->m_curr->pos);
1151 
1152  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, !m_evt_handler->m_curr->line_contents.rem.begins_with(' '), m_evt_handler->m_curr->pos);
1153  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, !m_evt_handler->m_curr->line_contents.rem.begins_with('\n'), m_evt_handler->m_curr->pos);
1154 
1155  if(!m_evt_handler->m_curr->line_contents.rem.len || !_is_valid_start_scalar_plain_flow(m_evt_handler->m_curr->line_contents.rem))
1156  return false;
1157 
1158  substr s = _buf().sub(m_evt_handler->m_curr->pos.offset);
1159  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.begins_with(m_evt_handler->m_curr->line_contents.rem), m_evt_handler->m_curr->pos);
1160 
1161  _c4dbgp("scanning mapflow scalar...");
1162 
1163  bool needs_filter = false;
1164  size_t col = 0; // zero-based column
1165  size_t offs = 0; // offset
1166  for( ; offs < s.len; ++offs, ++col)
1167  {
1168  const char c = s.str[offs];
1169  switch(c)
1170  {
1171  case ',':
1172  case '}':
1173  _c4dbgpf("found terminating character at {}: '{}'", offs, c);
1174  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, offs > 0, m_evt_handler->m_curr->pos);
1175  goto ended_scalar;
1176  case '\n':
1177  _c4dbgpf("found '\\n' at col={}", col);
1178  if(!_scan_scalar_plain_handle_newline(s, offs))
1179  goto ended_scalar;
1180  col = (size_t)-1; // so that col is 0 in the next loop iteration
1181  needs_filter = true;
1182  break;
1183  case '\r':
1184  --col; // don't count \r when calling _line_progressed()
1185  needs_filter = true;
1186  break;
1187  case ':':
1188  _c4dbgpf("found ':'", c);
1189  if(s.len == offs+1)
1190  break;
1191  {
1192  const char next = s.str[offs+1];
1193  _c4dbgpf("next='{}'", c);
1194  if(next == ' ' || next == ',' || next == '}' || next == '\n' || next == '\r' RYML_WITH_TAB_TOKENS_(|| next == '\t'))
1195  {
1196  _c4dbgpf("found terminating character: '{}'", c);
1197  goto ended_scalar;
1198  }
1199  }
1200  break;
1201  case '{':
1202  case '[':
1203  _line_progressed(col);
1204  _c4err("invalid character: '{}'", c); // noreturn
1205  break;
1206  case ']':
1207  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RSEQIMAP), m_evt_handler->m_curr->pos);
1208  goto ended_scalar;
1209  default:
1210  ;
1211  }
1212  }
1213 
1214 ended_scalar:
1215 
1216  _line_progressed(col);
1217  s = s.first(offs);
1218  sc->scalar = s.trimr(RYML_WITH_OR_WITHOUT_TAB_TOKENS_(" \t", ' '));
1219  sc->needs_filter = needs_filter;
1220 
1221  _c4prscalar("scanned plain scalar", sc->scalar, /*keep_newlines*/true);
1222 
1223  return sc->scalar.len > 0u;
1224 }
1225 
1226 template<class EventHandler>
1227 bool ParseEngine<EventHandler>::_scan_scalar_seq_json(ScannedScalar *C4_RESTRICT sc)
1228 {
1229  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RMAP), m_evt_handler->m_curr->pos);
1230  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RBLCK), m_evt_handler->m_curr->pos);
1231  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RSEQ), m_evt_handler->m_curr->pos);
1232  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RFLOW), m_evt_handler->m_curr->pos);
1233 
1234  substr s = m_evt_handler->m_curr->line_contents.rem;
1235  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, !s.begins_with(' '), m_evt_handler->m_curr->pos);
1236  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.len > 0, m_evt_handler->m_curr->pos);
1237 
1238  _c4dbgp("seq_json: scanning scalar...");
1239 
1240  switch(s.str[0])
1241  {
1242  case ']':
1243  case '{':
1244  case ',':
1245  _c4dbgp("seq_json: not a scalar.");
1246  return false;
1247  }
1248 
1249  {
1250  const size_t len = _begins_with_special_json_scalar(s);
1251  if(len)
1252  {
1253  char c = s.len > len ? s.str[len] : ',';
1254  if(c == ',' || c == ']' || c == ' ' || c == '\n' || c == '\t' || c == '\r')
1255  {
1256  sc->scalar = s.first(len);
1257  sc->needs_filter = false;
1258  _c4dbgpf("seq_json: special scalar: '{}'", sc->scalar);
1259  _line_progressed(len);
1260  return true;
1261  }
1262  else
1263  {
1264  return false;
1265  }
1266  }
1267  }
1268 
1269  // must be a number or special scalar
1270  size_t i = 0;
1271  for( ; i < s.len; ++i)
1272  {
1273  const char c = s.str[i];
1274  switch(c)
1275  {
1276  case ',':
1277  case ']':
1278  case ' ':
1279  case '\t':
1280  _c4dbgpf("seq_json: found terminating character: '{}'", c);
1281  goto ended_scalar;
1282  default:
1283  ;
1284  }
1285  }
1286 
1287 ended_scalar:
1288 
1289  _line_progressed(i);
1290  sc->scalar = s.first(i);
1291  sc->needs_filter = false;
1292  _c4dbgpf("seq_json: scalar was {}", prs_(sc->scalar, /*escape*/true));
1293 
1294  return true;
1295 }
1296 
1297 template<class EventHandler>
1298 bool ParseEngine<EventHandler>::_scan_scalar_map_json(ScannedScalar *C4_RESTRICT sc)
1299 {
1300  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RSEQ), m_evt_handler->m_curr->pos);
1301  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RBLCK), m_evt_handler->m_curr->pos);
1302  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RMAP), m_evt_handler->m_curr->pos);
1303  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RFLOW), m_evt_handler->m_curr->pos);
1304  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RKEY|RVAL), m_evt_handler->m_curr->pos);
1305 
1306  substr s = m_evt_handler->m_curr->line_contents.rem;
1307  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, !s.begins_with(' '), m_evt_handler->m_curr->pos);
1308  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.len > 0, m_evt_handler->m_curr->pos);
1309 
1310  _c4dbgp("scanning scalar...");
1311 
1312  {
1313  const size_t len = _begins_with_special_json_scalar(s);
1314  if(len)
1315  {
1316  char c = s.len > len ? s.str[len] : ',';
1317  _c4dbgpf("begins with special scalar: {} next='{}'", s.first(len), _c4prc(c));
1318  if(c == ',' || c == '}' || c == ' ' || c == '\n' || c == '\t' || c == '\r')
1319  {
1320  sc->scalar = s.first(len);
1321  sc->needs_filter = false;
1322  _c4dbgpf("special json scalar: '{}'", prs_(sc->scalar));
1323  _line_progressed(len);
1324  return true;
1325  }
1326  else
1327  {
1328  return false;
1329  }
1330  }
1331  }
1332 
1333  // must be a number
1334  size_t i = 0;
1335  for( ; i < s.len; ++i)
1336  {
1337  const char c = s.str[i];
1338  switch(c)
1339  {
1340  case ',':
1341  case '}':
1342  case ' ':
1343  case '\t':
1344  _c4dbgpf("found terminating character: '{}'", c);
1345  goto ended_scalar;
1346  default:
1347  ;
1348  }
1349  }
1350 
1351 ended_scalar:
1352 
1353  if C4_LIKELY(i > 0)
1354  {
1355  _line_progressed(i);
1356  sc->scalar = s.first(i);
1357  sc->needs_filter = false;
1358  _c4dbgpf("scalar was {}", prs_(sc->scalar));
1359  return true;
1360  }
1361 
1362  return false;
1363 }
1364 
1365 template<class EventHandler>
1366 bool ParseEngine<EventHandler>::_is_doc_begin(csubstr s)
1367 {
1368  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s[0] == '-', m_evt_handler->m_curr->pos);
1369  return (m_evt_handler->m_curr->line_contents.indentation == 0u && m_evt_handler->m_curr->at_line_beginning() && _is_doc_begin_token(s));
1370 }
1371 
1372 template<class EventHandler>
1373 bool ParseEngine<EventHandler>::_is_doc_end(csubstr s)
1374 {
1375  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s[0] == '.', m_evt_handler->m_curr->pos);
1376  return (m_evt_handler->m_curr->line_contents.indentation == 0u && m_evt_handler->m_curr->at_line_beginning() && _is_doc_end_token(s));
1377 }
1378 
1379 template<class EventHandler>
1380 bool ParseEngine<EventHandler>::_scan_scalar_plain_blck(ScannedScalar *C4_RESTRICT sc, size_t indentation)
1381 {
1382  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RFLOW), m_evt_handler->m_curr->pos);
1383  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RSEQIMAP), m_evt_handler->m_curr->pos);
1384  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RBLCK|RUNK|USTY), m_evt_handler->m_curr->pos);
1385 
1386  substr s = m_evt_handler->m_curr->line_contents.rem;
1387  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, !s.begins_with(' '), m_evt_handler->m_curr->pos);
1388  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.len > 0, m_evt_handler->m_curr->pos);
1389 
1390  switch(s.str[0])
1391  {
1392  case '-':
1393  if(_is_blck_token(s))
1394  {
1395  return false;
1396  }
1397  else if(_is_doc_begin(s))
1398  {
1399  _c4dbgp("token is doc start");
1400  return false;
1401  }
1402  break;
1403  case ':':
1404  case '?':
1405  if(_is_blck_token(s))
1406  return false;
1407  break;
1408  case '[':
1409  case '{':
1410  case '&':
1411  case '*':
1412  case '!':
1413  case '\t':
1414  case ',':
1415  case '%':
1416  return false;
1417  case '.':
1418  if(_is_doc_end(s))
1419  {
1420  _c4dbgp("token is doc end");
1421  return false;
1422  }
1423  break;
1424  }
1425 
1426  _c4dbgpf("plain scalar! indentation={}", indentation);
1427 
1428  const size_t start_offset = m_evt_handler->m_curr->pos.offset;
1429  const size_t start_line = m_evt_handler->m_curr->pos.line;
1430 
1431  bool needs_filter = false;
1432  while(true)
1433  {
1434  _c4dbgpf("plain scalar line: {}", prs_(s));
1435  for(size_t i = 0; i < s.len; ++i)
1436  {
1437  const char curr = s.str[i];
1438  //_c4dbgpf("[{}]='{}'", i, _c4prc(curr));
1439  switch(curr)
1440  {
1441  case ':':
1442  _c4dbgpf("[{}]: got suspicious ':'", i);
1443  // are there more characters?
1444  if((i + 1 == s.len) || ((s.str[i+1] == ' ') RYML_WITH_TAB_TOKENS_( || (s.str[i+1] == '\t'))))
1445  {
1446  _c4dbgpf("followed by '{}'", i+1 == s.len ? csubstr("\\n") : _c4prc(s.str[i+1]));
1447  _line_progressed(i);
1448  // ': ' is accepted only on the first line
1449  if C4_LIKELY(m_evt_handler->m_curr->pos.line == start_line)
1450  {
1451  _c4dbgp("start line. scalar ends here");
1452  goto ended_scalar;
1453  }
1454  else
1455  {
1456  _c4err("multiline scalars cannot be used as keys");
1457  }
1458  }
1459  else
1460  {
1461  size_t j = i;
1462  while(j + 1 < s.len && s.str[j+1] == ':')
1463  {
1464  _c4dbgp("skip colon");
1465  ++j;
1466  }
1467  i = j > i ? j-1 : i;
1468  _c4dbgp("nothing to see here");
1469  }
1470  break;
1471  case '#':
1472  _c4dbgp("got suspicious '#'");
1473  if(!i || (s.str[i-1] == ' ' || s.str[i-1] == '\t'))
1474  {
1475  _c4dbgp("comment! scalar ends here");
1476  _line_progressed(i);
1477  goto ended_scalar;
1478  }
1479  else
1480  {
1481  _c4dbgp("nothing to see here");
1482  }
1483  break;
1484  }
1485  }
1486  _line_progressed(s.len);
1487  csubstr next_peeked = _peek_next_line(m_evt_handler->m_curr->pos.offset);
1488  next_peeked = next_peeked.trimr("\n\r");
1489  const size_t next_indentation = next_peeked.first_not_of(' ');
1490  _c4dbgpf("indentation curr={} next={}", indentation, next_indentation);
1491  if(next_indentation < indentation)
1492  {
1493  _c4dbgp("smaller indentation! scalar ended");
1494  goto ended_scalar;
1495  }
1496  else if(next_indentation == 0 && next_peeked.len > 0)
1497  {
1498  const char first = next_peeked.str[0];
1499  switch(first)
1500  {
1501  case '-':
1502  _c4dbgpf("doc begin? peeked={}", prs_(next_peeked, size_t(3)));
1503  if(_is_doc_begin_token(next_peeked))
1504  {
1505  _c4dbgp("doc begin! scalar ended");
1506  goto ended_scalar;
1507  }
1508  break;
1509  case '.':
1510  _c4dbgpf("doc end? peeked={}", prs_(next_peeked, size_t(3)));
1511  if(_is_doc_end_token(next_peeked))
1512  {
1513  _c4dbgp("doc end! scalar ended");
1514  goto ended_scalar;
1515  }
1516  break;
1517  }
1518  }
1519  // load with next line
1520  _c4dbgp("next line!");
1521  if(!_finished_file())
1522  {
1523  _c4dbgp("next line!");
1524  _line_ended();
1525  _scan_line();
1526  }
1527  else
1528  {
1529  _c4dbgp("file finished!");
1530  goto ended_scalar;
1531  }
1532  s = m_evt_handler->m_curr->line_contents.rem;
1533  needs_filter = true;
1534  }
1535 
1536 ended_scalar:
1537 
1538  sc->scalar = _buf().range(start_offset, m_evt_handler->m_curr->pos.offset).trimr(" \n\r\t");
1539  sc->needs_filter = needs_filter;
1540 
1541  _c4dbgpf("scalar was {}", prs_(sc->scalar));
1542 
1543  return true;
1544 }
1545 
1546 template<class EventHandler>
1547 C4_ALWAYS_INLINE bool ParseEngine<EventHandler>::_scan_scalar_plain_seq_blck(ScannedScalar *C4_RESTRICT sc) // LCOV_EXCL_LINE
1548 {
1549  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RMAP), m_evt_handler->m_curr->pos);
1550  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RFLOW), m_evt_handler->m_curr->pos);
1551  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RSEQIMAP), m_evt_handler->m_curr->pos);
1552  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RSEQ), m_evt_handler->m_curr->pos);
1553  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RBLCK), m_evt_handler->m_curr->pos);
1554  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RVAL), m_evt_handler->m_curr->pos);
1555  return _scan_scalar_plain_blck(sc, m_evt_handler->m_curr->indref + 1u);
1556 }
1557 
1558 template<class EventHandler>
1559 C4_ALWAYS_INLINE bool ParseEngine<EventHandler>::_scan_scalar_plain_map_blck(ScannedScalar *C4_RESTRICT sc) // LCOV_EXCL_LINE
1560 {
1561  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RSEQ), m_evt_handler->m_curr->pos);
1562  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RFLOW), m_evt_handler->m_curr->pos);
1563  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RMAP), m_evt_handler->m_curr->pos);
1564  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RBLCK), m_evt_handler->m_curr->pos);
1565  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RKEY|RVAL|QMRK), m_evt_handler->m_curr->pos);
1566  return _scan_scalar_plain_blck(sc, m_evt_handler->m_curr->indref + 1u);
1567 }
1568 
1569 template<class EventHandler>
1570 C4_ALWAYS_INLINE bool ParseEngine<EventHandler>::_scan_scalar_plain_unk(ScannedScalar *C4_RESTRICT sc) // LCOV_EXCL_LINE
1571 {
1572  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RUNK|USTY), m_evt_handler->m_curr->pos);
1573  return _scan_scalar_plain_blck(sc, m_evt_handler->m_curr->indref);
1574 }
1575 
1576 
1577 //-----------------------------------------------------------------------------
1578 
1579 template<class EventHandler>
1580 substr ParseEngine<EventHandler>::_peek_next_line(size_t pos) const
1581 {
1582  substr rem{}; // declare here because of the goto
1583  size_t nlpos{}; // declare here because of the goto
1584  pos = pos == npos ? m_evt_handler->m_curr->pos.offset : pos;
1585  if(pos >= _buf().len)
1586  goto next_is_empty;
1587 
1588  // look for the next newline chars, and jump to the right of those
1589  rem = _from_next_line(_buf().sub(pos));
1590  if(rem.empty())
1591  goto next_is_empty;
1592 
1593  // now get everything up to and including the following newline chars
1594  nlpos = rem.first_of("\r\n");
1595  if((nlpos != csubstr::npos) && (nlpos + 1 < rem.len))
1596  nlpos += _extend_from_combined_newline(rem[nlpos], rem[nlpos+1]);
1597  rem = rem.left_of(nlpos, /*include_pos*/true);
1598 
1599  _c4dbgpf("peek next line @ {}: (len={})'{}'", pos, rem.len, rem.trimr("\r\n"));
1600  return rem;
1601 
1602 next_is_empty:
1603  _c4dbgpf("peek next line @ {}: (len=0)''", pos);
1604  return rem;
1605 }
1606 
1607 //-----------------------------------------------------------------------------
1608 
1609 template<class EventHandler>
1610 void ParseEngine<EventHandler>::_scan_line()
1611 {
1612  if C4_LIKELY(m_evt_handler->m_curr->pos.offset < _buf().len)
1613  m_evt_handler->m_curr->line_contents.reset_with_next_line(_buf(), m_evt_handler->m_curr->pos.offset);
1614  else
1615  m_evt_handler->m_curr->line_contents.reset_with_next_line(_buf().last(0), 0);
1616 }
1617 
1618 template<class EventHandler>
1619 void ParseEngine<EventHandler>::_line_progressed(size_t ahead)
1620 {
1621  _c4dbgpf("line[{}] ({} cols) progressed by {}: col {}-->{} offset {}-->{}",
1622  m_evt_handler->m_curr->pos.line,
1623  m_evt_handler->m_curr->line_contents.full.len,
1624  ahead, m_evt_handler->m_curr->pos.col,
1625  m_evt_handler->m_curr->pos.col+ahead,
1626  m_evt_handler->m_curr->pos.offset,
1627  m_evt_handler->m_curr->pos.offset+ahead);
1628  m_evt_handler->m_curr->pos.offset += ahead;
1629  m_evt_handler->m_curr->pos.col += ahead;
1630  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->pos.col <= m_evt_handler->m_curr->line_contents.num_cols+1, m_evt_handler->m_curr->pos);
1631  m_evt_handler->m_curr->line_contents.rem = m_evt_handler->m_curr->line_contents.rem.sub(ahead);
1632 }
1633 
1634 template<class EventHandler>
1635 void ParseEngine<EventHandler>::_line_ended()
1636 {
1637  _c4dbgpf("line[{}] ({} cols) ended! offset {}-->{} / col {}-->{}",
1638  m_evt_handler->m_curr->pos.line,
1639  m_evt_handler->m_curr->line_contents.full.len,
1640  m_evt_handler->m_curr->pos.offset, m_evt_handler->m_curr->pos.offset + m_evt_handler->m_curr->line_contents.full.len - m_evt_handler->m_curr->line_contents.num_cols,
1641  m_evt_handler->m_curr->pos.col, 1);
1642  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->pos.col == m_evt_handler->m_curr->line_contents.num_cols + 1, m_evt_handler->m_curr->pos);
1643  m_evt_handler->m_curr->pos.offset += m_evt_handler->m_curr->line_contents.full.len - m_evt_handler->m_curr->line_contents.num_cols;
1644  ++m_evt_handler->m_curr->pos.line;
1645  m_evt_handler->m_curr->pos.col = 1;
1646 }
1647 
1648 template<class EventHandler>
1649 void ParseEngine<EventHandler>::_line_ended_undo()
1650 {
1651  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->pos.col == 1u, m_evt_handler->m_curr->pos);
1652  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->pos.line > 0u, m_evt_handler->m_curr->pos);
1653  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->pos.offset >= m_evt_handler->m_curr->line_contents.full.len - m_evt_handler->m_curr->line_contents.num_cols, m_evt_handler->m_curr->pos);
1654  const size_t delta = m_evt_handler->m_curr->line_contents.full.len - m_evt_handler->m_curr->line_contents.num_cols;
1655  _c4dbgpf("line[{}] undo ended! line {}-->{}, offset {}-->{}", m_evt_handler->m_curr->pos.line, m_evt_handler->m_curr->pos.line, m_evt_handler->m_curr->pos.line - 1, m_evt_handler->m_curr->pos.offset, m_evt_handler->m_curr->pos.offset - delta);
1656  m_evt_handler->m_curr->pos.offset -= delta;
1657  --m_evt_handler->m_curr->pos.line;
1658  m_evt_handler->m_curr->pos.col = m_evt_handler->m_curr->line_contents.num_cols + 1u;
1659  // don't forget to undo also the changes to the remainder of the line
1660  //RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->pos.offset >= _buf().len || _buf()[m_evt_handler->m_curr->pos.offset] == '\n' || _buf()[m_evt_handler->m_curr->pos.offset] == '\r', m_evt_handler->m_curr->pos);
1661  m_evt_handler->m_curr->line_contents.rem = _buf().sub(m_evt_handler->m_curr->pos.offset, 0);
1662 }
1663 
1664 
1665 //-----------------------------------------------------------------------------
1666 template<class EventHandler>
1667 void ParseEngine<EventHandler>::_set_indentation(size_t indentation) noexcept
1668 {
1669  m_evt_handler->m_curr->indref = indentation;
1670  _c4dbgpf("state[{}]: saving indentation: {}", m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref);
1671 }
1672 
1673 template<class EventHandler>
1674 void ParseEngine<EventHandler>::_save_indentation()
1675 {
1676  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->line_contents.rem.is_sub(m_evt_handler->m_curr->line_contents.full), m_evt_handler->m_curr->pos);
1677  m_evt_handler->m_curr->indref = m_evt_handler->m_curr->line_contents.current_col();
1678  _c4dbgpf("state[{}]: saving indentation: {}", m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref);
1679 }
1680 
1681 template<class EventHandler>
1682 void ParseEngine<EventHandler>::_mark_seqflow_val_end() noexcept
1683 {
1684  _c4dbgpf("SEQFLOW. mark val end at line={}", m_evt_handler->m_curr->pos.line);
1685  m_prev_val_end = m_evt_handler->m_curr->pos.line;
1686 }
1687 
1688 
1689 //-----------------------------------------------------------------------------
1690 
1691 template<class EventHandler>
1692 void ParseEngine<EventHandler>::_flow_container_was_a_key(size_t orig_indent)
1693 {
1694  _c4dbgpf("flow container is followed by colon! orig_indent={}", orig_indent);
1695  m_evt_handler->actually_val_is_first_key_of_new_map_block();
1696  addrem_flags(RMAP|RVAL|RBLCK, RKCL|RSEQ|RUNK);
1697  _set_indentation(orig_indent);
1698  _maybe_skip_whitespace_tokens();
1699 }
1700 
1701 template<class EventHandler>
1702 void ParseEngine<EventHandler>::_end_flow_container(size_t orig_indent, bool multiline)
1703 {
1704  // this is called AFTER ending the flow container,
1705  // so now we're at the parent container's scope
1706  if(has_all(RMAP|RBLCK) && has_none(RKCL|RVAL|RNXT))
1707  {
1708  _c4dbgp("flow container: end as vanilla block map key!");
1709  if C4_UNLIKELY(multiline)
1710  _c4err("multiline key is invalid");
1711  if C4_UNLIKELY(!_maybe_scan_following_colon())
1712  _c4err("could not find ':' colon after key");
1713  _maybe_skip_whitespace_tokens();
1714  addrem_flags(RVAL, RKEY|RKCL|RNXT);
1715  }
1716  else if(has_none(RFLOW))
1717  {
1718  _c4dbgp("end_flow_container: now not in flow!");
1719  if(has_any(RUNK|RSEQ|RKCL) && _maybe_scan_following_colon())
1720  {
1721  if C4_UNLIKELY(multiline)
1722  _c4err("multiline key is invalid");
1723  _flow_container_was_a_key(orig_indent);
1724  }
1725  else
1726  {
1727  _c4dbgp("end_flow_container: end map as key!");
1728  }
1729  }
1730  else if(has_any(RSEQ))
1731  {
1732  _c4dbgp("end_flow_container: now in a flow seq");
1733  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RFLOW), m_evt_handler->m_curr->pos);
1734  _mark_seqflow_val_end();
1735  }
1736 }
1737 
1738 template<class EventHandler>
1739 void ParseEngine<EventHandler>::_end_map_flow()
1740 {
1741  bool multiline = m_evt_handler->m_parent->pos.line < m_evt_handler->m_curr->pos.line;
1742  size_t orig_indent = m_evt_handler->m_curr->indref;
1743  _c4dbgpf("mapflow: end, multiline={}", multiline);
1744  m_evt_handler->end_map_flow(multiline && m_options.detect_flow_ml(), m_options.flow_ml_style().m_bits);
1745  _end_flow_container(orig_indent, multiline);
1746 }
1747 
1748 template<class EventHandler>
1749 void ParseEngine<EventHandler>::_end_seq_flow()
1750 {
1751  bool multiline = m_evt_handler->m_parent->pos.line < m_evt_handler->m_curr->pos.line;
1752  size_t orig_indent = m_evt_handler->m_curr->indref;
1753  _c4dbgpf("seqflow: end, multiline={}", multiline);
1754  m_evt_handler->end_seq_flow(multiline && m_options.detect_flow_ml(), m_options.flow_ml_style().m_bits);
1755  _end_flow_container(orig_indent, multiline);
1756 }
1757 
1758 template<class EventHandler>
1759 void ParseEngine<EventHandler>::_end_map_blck()
1760 {
1761  _c4dbgp("mapblck: end");
1762  if(has_any(RKCL|RVAL))
1763  {
1764  _c4dbgp("mapblck: set missing val");
1765  _handle_annotations_before_blck_val_scalar();
1766  m_evt_handler->set_val_scalar_plain_empty();
1767  }
1768  else if(has_any(QMRK))
1769  {
1770  _c4dbgp("mapblck: set missing keyval");
1771  _handle_annotations_before_blck_key_scalar();
1772  m_evt_handler->set_key_scalar_plain_empty();
1773  _handle_annotations_before_blck_val_scalar();
1774  m_evt_handler->set_val_scalar_plain_empty();
1775  }
1776  m_evt_handler->end_map_block();
1777 }
1778 
1779 template<class EventHandler>
1780 void ParseEngine<EventHandler>::_end_seq_blck()
1781 {
1782  if(has_any(RVAL))
1783  {
1784  _c4dbgp("seqblck: set missing val");
1785  _handle_annotations_before_blck_val_scalar();
1786  m_evt_handler->set_val_scalar_plain_empty();
1787  }
1788  m_evt_handler->end_seq_block();
1789 }
1790 
1791 template<class EventHandler>
1792 void ParseEngine<EventHandler>::_end2_map()
1793 {
1794  _c4dbgp("map: end");
1795  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RMAP), m_evt_handler->m_curr->pos);
1796  if(has_any(RBLCK))
1797  {
1798  _end_map_blck();
1799  }
1800  else
1801  {
1802  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RFLOW), m_evt_handler->m_curr->pos);
1803  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(USTY), m_evt_handler->m_curr->pos);
1804  m_evt_handler->_pop();
1805  }
1806 }
1807 
1808 template<class EventHandler>
1809 void ParseEngine<EventHandler>::_end2_seq()
1810 {
1811  _c4dbgp("seq: end");
1812  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RSEQ), m_evt_handler->m_curr->pos);
1813  if(has_any(RBLCK))
1814  {
1815  _end_seq_blck();
1816  }
1817  else
1818  {
1819  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RFLOW), m_evt_handler->m_curr->pos);
1820  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(USTY), m_evt_handler->m_curr->pos);
1821  m_evt_handler->_pop();
1822  }
1823 }
1824 
1825 template<class EventHandler>
1826 void ParseEngine<EventHandler>::_begin2_doc()
1827 {
1828  _c4dbgp("begin_doc");
1829  m_has_directives_yaml = false;
1830  m_has_directives = false;
1831  m_doc_empty = true;
1832  add_flags(RDOC);
1833  m_evt_handler->begin_doc();
1834  m_evt_handler->m_curr->indref = 0; // ?
1835 }
1836 
1837 template<class EventHandler>
1838 void ParseEngine<EventHandler>::_begin2_doc_expl()
1839 {
1840  _c4dbgp("begin_doc_expl");
1841  m_has_directives_yaml = false;
1842  m_has_directives = false;
1843  m_doc_empty = true;
1844  add_flags(RDOC);
1845  m_evt_handler->begin_doc_expl();
1846  m_evt_handler->m_curr->indref = 0; // ?
1847 }
1848 
1849 template<class EventHandler>
1850 void ParseEngine<EventHandler>::_end2_doc()
1851 {
1852  _c4dbgp("doc: end");
1853  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RDOC), m_evt_handler->m_curr->pos);
1854  if(m_doc_empty || (m_pending_tags.num_entries || m_pending_anchors.num_entries))
1855  {
1856  _c4dbgp("doc was empty; add empty val");
1857  _handle_annotations_before_blck_val_scalar();
1858  m_evt_handler->set_val_scalar_plain_empty();
1859  }
1860  m_evt_handler->end_doc();
1861  m_bom_len = 0;
1862 }
1863 
1864 template<class EventHandler>
1865 void ParseEngine<EventHandler>::_end2_doc_expl()
1866 {
1867  _c4dbgp("doc: end");
1868  if(m_doc_empty || (m_pending_tags.num_entries || m_pending_anchors.num_entries))
1869  {
1870  _c4dbgp("doc: no children; add empty val");
1871  _handle_annotations_before_blck_val_scalar();
1872  m_evt_handler->set_val_scalar_plain_empty();
1873  }
1874  m_evt_handler->end_doc_expl();
1875  m_bom_len = 0;
1876 }
1877 
1878 template<class EventHandler>
1879 void ParseEngine<EventHandler>::_maybe_begin_doc()
1880 {
1881  if(has_none(RDOC))
1882  {
1883  _c4dbgp("doc must be started");
1884  _begin2_doc();
1885  }
1886 }
1887 template<class EventHandler>
1888 void ParseEngine<EventHandler>::_maybe_end_doc()
1889 {
1890  if(has_any(RDOC))
1891  {
1892  _c4dbgp("doc must be finished");
1893  _end2_doc();
1894  }
1895  else if(m_doc_empty && (m_pending_tags.num_entries || m_pending_anchors.num_entries))
1896  {
1897  _c4dbgp("no doc to finish, but pending annotations");
1898  m_evt_handler->begin_doc();
1899  _handle_annotations_before_blck_val_scalar();
1900  m_evt_handler->set_val_scalar_plain_empty();
1901  m_evt_handler->end_doc();
1902  }
1903 }
1904 
1905 template<class EventHandler>
1906 void ParseEngine<EventHandler>::_end_doc_suddenly__pop()
1907 {
1908  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_stack.size() >= 1, m_evt_handler->m_curr->pos);
1909  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_stack[0].flags & RDOC, m_evt_handler->m_curr->pos);
1910  _c4dbgp("root is RDOC");
1911  if(m_evt_handler->m_curr->level != 0)
1912  _handle_indentation_pop(&m_evt_handler->m_stack[0]);
1913  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RDOC), m_evt_handler->m_curr->pos);
1914 }
1915 
1916 /** Check whether the current parse tokens are trailing on the
1917  * previous doc, and raise an error if they are */
1918 template<class EventHandler>
1919 void ParseEngine<EventHandler>::_check_trailing_doc_token()
1920 {
1921  const bool is_root = (m_evt_handler->m_stack.size() == 1u);
1922  const bool isndoc = (m_evt_handler->m_curr->flags & NDOC) != 0;
1923  const bool suspicious = m_evt_handler->template has_any_<MAP|SEQ|VAL>();
1924  _c4dbgpf("target={} isroot={} suspicious={} ndoc={}", m_evt_handler->m_curr->node_id, is_root, suspicious, isndoc);
1925  if((is_root || m_evt_handler->template has_any_<DOC>()) && suspicious && !isndoc)
1926  _c4err("parse error");
1927 }
1928 
1929 template<class EventHandler>
1930 void ParseEngine<EventHandler>::_end_doc_suddenly()
1931 {
1932  _c4dbgp("end doc suddenly");
1933  _end_doc_suddenly__pop();
1934  _end2_doc_expl();
1935  addrem_flags(RUNK|RTOP|NDOC, RMAP|RSEQ|RDOC);
1936 }
1937 
1938 template<class EventHandler>
1939 void ParseEngine<EventHandler>::_check_doc_end_tokens() const
1940 {
1941  csubstr rem = m_evt_handler->m_curr->line_contents.rem;
1942  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, !rem.begins_with_any(". \t"), m_evt_handler->m_curr->pos);
1943  if C4_UNLIKELY(rem.len && !rem.begins_with('#'))
1944  {
1945  _c4err("parse error");
1946  }
1947 }
1948 
1949 template<class EventHandler>
1950 void ParseEngine<EventHandler>::_start_doc_suddenly()
1951 {
1952  _c4dbgp("start doc suddenly");
1953  _end_doc_suddenly__pop();
1954  _end2_doc();
1955  _begin2_doc_expl();
1956 }
1957 
1958 template<class EventHandler>
1959 void ParseEngine<EventHandler>::_end_stream()
1960 {
1961  _c4dbgpf("end_stream, level={} node_id={}", m_evt_handler->m_curr->level, m_evt_handler->m_curr->node_id);
1962  if C4_UNLIKELY(has_all(RSEQ|RFLOW))
1963  _c4err("missing terminating ]");
1964  else if C4_UNLIKELY(has_all(RMAP|RFLOW))
1965  _c4err("missing terminating }");
1966  if(m_evt_handler->m_stack.size() > 1)
1967  _handle_indentation_pop(m_evt_handler->m_stack.begin());
1968  if(has_all(RDOC))
1969  {
1970  _end2_doc();
1971  }
1972  else if(has_all(RTOP|RUNK))
1973  {
1974  if(m_pending_anchors.num_entries || m_pending_tags.num_entries)
1975  {
1976  if(m_doc_empty)
1977  {
1978  m_evt_handler->begin_doc();
1979  _handle_annotations_before_blck_val_scalar();
1980  m_evt_handler->set_val_scalar_plain_empty();
1981  m_evt_handler->end_doc();
1982  }
1983  }
1984  }
1985  m_evt_handler->end_stream();
1986  if C4_UNLIKELY(m_has_directives)
1987  _c4err("directives cannot be used without a document");
1988 }
1989 
1990 
1991 template<class EventHandler>
1992 void ParseEngine<EventHandler>::_handle_indentation_pop(ParserState const* popto)
1993 {
1994  _c4dbgpf("popping {} level{}: from level {}(@ind={}) to level {}(@ind={})", m_evt_handler->m_curr->level - popto->level, (((m_evt_handler->m_curr->level - popto->level) > 1) ? "s" : ""), m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref, popto->level, popto->indref);
1995  while(m_evt_handler->m_curr != popto)
1996  {
1997  if(has_any(RSEQ))
1998  {
1999  _c4dbgpf("popping seq at level {} (indentation={},addr={})", m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref, m_evt_handler->m_curr);
2000  _end2_seq();
2001  }
2002  else if(has_any(RMAP))
2003  {
2004  _c4dbgpf("popping map at level {} (indentation={},addr={})", m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref, m_evt_handler->m_curr);
2005  _end2_map();
2006  }
2007  else
2008  {
2009  break;
2010  }
2011  }
2012  _c4dbgpf("current level is {} (indentation={})", m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref);
2013 }
2014 
2015 template<class EventHandler>
2016 void ParseEngine<EventHandler>::_handle_indentation_pop_from_block_seq()
2017 {
2018  // search the stack frame to jump to based on its indentation
2019  using state_type = typename EventHandler::state;
2020  state_type const* popto = nullptr;
2021  auto &stack = m_evt_handler->m_stack;
2022  RYML_ASSERT_PARSE_CB_(stack.m_callbacks, stack.is_contiguous(), m_evt_handler->m_curr->pos); // this search relies on the stack being contiguous
2023  RYML_ASSERT_PARSE_CB_(stack.m_callbacks, m_evt_handler->m_curr >= stack.begin() && m_evt_handler->m_curr < stack.end(), m_evt_handler->m_curr->pos);
2024  const size_t ind = m_evt_handler->m_curr->line_contents.indentation;
2025  #ifdef RYML_DBG
2026  _print_state_stack();
2027  #endif
2028  for(state_type const* s = m_evt_handler->m_curr-1; s >= stack.begin(); --s)
2029  {
2030  _c4dbgpf("searching for state with indentation {}. curr={} (level={},node={})", ind, s->indref, s->level, s->node_id);
2031  if(s->indref == ind)
2032  {
2033  _c4dbgpf("gotit!!! level={} node={}", s->level, s->node_id);
2034  popto = s;
2035  break;
2036  }
2037  }
2038  if(!popto || popto >= m_evt_handler->m_curr || popto->level >= m_evt_handler->m_curr->level)
2039  {
2040  _c4err("parse error: incorrect indentation?");
2041  }
2042  _handle_indentation_pop(popto);
2043 }
2044 
2045 template<class EventHandler>
2046 void ParseEngine<EventHandler>::_handle_indentation_pop_from_block_map()
2047 {
2048  // search the stack frame to jump to based on its indentation
2049  using state_type = typename EventHandler::state;
2050  auto &stack = m_evt_handler->m_stack;
2051  RYML_ASSERT_PARSE_CB_(stack.m_callbacks, stack.is_contiguous(), m_evt_handler->m_curr->pos); // this search relies on the stack being contiguous
2052  RYML_ASSERT_PARSE_CB_(stack.m_callbacks, m_evt_handler->m_curr >= stack.begin() && m_evt_handler->m_curr < stack.end(), m_evt_handler->m_curr->pos);
2053  const size_t ind = m_evt_handler->m_curr->line_contents.indentation;
2054  state_type const* popto = nullptr;
2055  #ifdef RYML_DBG
2056  char flagbuf_[128];
2057  _print_state_stack(flagbuf_);
2058  #endif
2059  for(state_type const* s = m_evt_handler->m_curr-1; s > stack.begin(); --s) // never go to the stack bottom. that's the root
2060  {
2061  _c4dbgpf("searching for state with indentation {}. current: ind={},level={},node={},flags={}", ind, s->indref, s->level, s->node_id, detail::_parser_flags_to_str(flagbuf_, s->flags));
2062  if(s->indref < ind)
2063  {
2064  break;
2065  }
2066  else if(s->indref == ind)
2067  {
2068  _c4dbgpf("same indentation!!! level={} node={}", s->level, s->node_id);
2069  if(popto && has_any(RTOP, s) && has_none(RMAP|RSEQ, s))
2070  {
2071  break;
2072  }
2073  popto = s;
2074  if(has_all(RSEQ|RBLCK, s))
2075  {
2076  csubstr rem = m_evt_handler->m_curr->line_contents.rem;
2077  const size_t first = rem.first_not_of(' ');
2078  RYML_ASSERT_PARSE_CB_(stack.m_callbacks, first == ind || first == npos, m_evt_handler->m_curr->pos);
2079  rem = rem.right_of(first, true);
2080  _c4dbgpf("indentless? rem='{}' first={}", rem, first);
2081  if(rem.begins_with('-') && _is_blck_token(rem))
2082  {
2083  _c4dbgp("parent was indentless seq");
2084  break;
2085  }
2086  }
2087  }
2088  }
2089  if(!popto || popto >= m_evt_handler->m_curr || popto->level >= m_evt_handler->m_curr->level)
2090  {
2091  _c4err("parse error: incorrect indentation?");
2092  }
2093  _handle_indentation_pop(popto);
2094 }
2095 
2096 
2097 //-----------------------------------------------------------------------------
2098 template<class EventHandler>
2099 void ParseEngine<EventHandler>::_check_valid_newline_in_quoted_scalar()
2100 {
2101  if C4_UNLIKELY(has_all(RMAP|RBLCK|RKEY))
2102  {
2103  _c4err("multiline quoted keys are invalid");
2104  }
2105  else // check contextual indentation
2106  {
2107  const size_t minindent = m_evt_handler->m_curr->indref + ((has_any(RMAP|RSEQ) && has_any(RBLCK)));
2108  _c4dbgpf("indent={} vs minindent={} indref={}", m_evt_handler->m_curr->line_contents.indentation, minindent, m_evt_handler->m_curr->indref);
2109  if(m_evt_handler->m_curr->line_contents.indentation < minindent)
2110  {
2111  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks,
2112  m_evt_handler->m_curr->line_contents.indentation == m_evt_handler->m_curr->line_contents.rem.first_not_of(' '),
2113  m_evt_handler->m_curr->pos);
2114  csubstr trimmed = m_evt_handler->m_curr->line_contents.rem.sub(m_evt_handler->m_curr->line_contents.indentation);
2115  _c4dbgpf("trimmed.len={} line={}", trimmed.len, prs_(m_evt_handler->m_curr->line_contents.rem, true));
2116  if C4_UNLIKELY(!!trimmed.len)
2117  {
2118  _c4err("bad indentation");
2119  }
2120  }
2121  }
2122 }
2123 
2124 
2125 //-----------------------------------------------------------------------------
2126 template<class EventHandler>
2127 ScannedScalar ParseEngine<EventHandler>::_scan_scalar_squot()
2128 {
2129  // quoted scalars can spread over multiple lines!
2130  // nice explanation here: http://yaml-multiline.info/
2131 
2132  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, _buf().sub(m_evt_handler->m_curr->pos.offset).begins_with('\''), m_evt_handler->m_curr->pos);
2133 
2134  // a span to the end of the file, skipping the opening quote
2135  substr s = _buf().sub(m_evt_handler->m_curr->pos.offset + 1);
2136  _line_progressed(1); // advance over the opening quote
2137  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, !m_evt_handler->m_curr->at_line_beginning(), m_evt_handler->m_curr->pos);
2138 
2139  bool needs_filter = false;
2140  size_t pos = npos; // find the pos of the matching quote
2141  while( ! _finished_file())
2142  {
2143  const csubstr line = m_evt_handler->m_curr->line_contents.rem;
2144  _c4dbgpf("scanning single quoted scalar @ line[{}]: {}", m_evt_handler->m_curr->pos.line, prs_(line));
2145  if C4_UNLIKELY(m_evt_handler->m_curr->at_line_beginning() && _is_doc_token(line))
2146  _c4err("token can not appear at line begin");
2147  for(size_t i = 0; i < line.len; ++i)
2148  {
2149  const char curr = line.str[i];
2150  if(curr == '\'') // single quotes are escaped with two single quotes
2151  {
2152  const char next = i+1 < line.len ? line.str[i+1] : '~';
2153  if(next != '\'') // so just look for the first quote
2154  { // without another after it
2155  _line_progressed(i + 1); // progress beyond the quote
2156  pos = i + (size_t)(line.str - s.str); // set pos to before the quote
2157  goto found_close;
2158  }
2159  else
2160  {
2161  needs_filter = true; // needs filter to remove escaped quotes
2162  ++i; // skip the escaped quote
2163  }
2164  }
2165  }
2166 
2167  needs_filter = true;
2168  _line_progressed(line.len);
2169  _line_ended();
2170  _scan_line();
2171  _check_valid_newline_in_quoted_scalar();
2172  }
2173 
2174  _c4err("reached end of file while looking for closing quote");
2175 
2176 found_close:
2177 
2178  _c4dbgpf("found closing quote at: {}", pos);
2179  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, pos != npos, m_evt_handler->m_curr->pos);
2180  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, pos >= 0, m_evt_handler->m_curr->pos);
2181  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.end() >= _buf().begin() && s.end() <= _buf().end(), m_evt_handler->m_curr->pos);
2182  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.end() == _buf().end() || *s.end() == '\'', m_evt_handler->m_curr->pos);
2183  _set_first_strict(s, pos);
2184 
2185  _c4prscalar("scanned squoted scalar", s, /*keep_newlines*/true);
2186 
2187  return ScannedScalar { s, needs_filter };
2188 }
2189 
2190 
2191 //-----------------------------------------------------------------------------
2192 template<class EventHandler>
2193 ScannedScalar ParseEngine<EventHandler>::_scan_scalar_dquot()
2194 {
2195  // quoted scalars can spread over multiple lines!
2196  // nice explanation here: http://yaml-multiline.info/
2197 
2198  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, _buf().sub(m_evt_handler->m_curr->pos.offset).begins_with('"'), m_evt_handler->m_curr->pos);
2199 
2200  // a span to the end of the file, skipping the opening quote
2201  substr s = _buf().sub(m_evt_handler->m_curr->pos.offset + 1);
2202  _line_progressed(1); // advance over the opening quote
2203  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, !m_evt_handler->m_curr->at_line_beginning(), m_evt_handler->m_curr->pos);
2204 
2205  bool needs_filter = false;
2206  size_t pos = npos; // find the pos of the matching quote
2207  while( ! _finished_file())
2208  {
2209  #if defined(__GNUC__) && (/*__GNUC__ == 12 || */__GNUC__ == 13)
2210  C4_DONT_OPTIMIZE(m_evt_handler->m_curr->line_contents.rem); // prevent hoisting
2211  #endif
2212  csubstr rem = m_evt_handler->m_curr->line_contents.rem;
2213  _c4dbgpf("scanning double quoted scalar @ line[{}]: line='{}'", m_evt_handler->m_curr->pos.line, rem);
2214  if C4_UNLIKELY(m_evt_handler->m_curr->at_line_beginning() && _is_doc_token(rem))
2215  _c4err("token can not appear at line begin");
2216  for(size_t i = 0; i < rem.len; ++i)
2217  {
2218  const char curr = rem.str[i];
2219  // every \ is an escape
2220  if(curr == '\\')
2221  {
2222  const char next = i+1 < rem.len ? rem.str[i+1] : '~';
2223  needs_filter = true;
2224  if(next == '"' || next == '\\')
2225  ++i;
2226  }
2227  else if(curr == '"')
2228  {
2229  _line_progressed(i + 1); // progress beyond the quote
2230  pos = i + (size_t)(rem.str - s.str); // set pos to before the quote
2231  goto found_close;
2232  }
2233  }
2234 
2235  // leading whitespace also needs filtering
2236  needs_filter = true;
2237  _line_progressed(rem.len);
2238  _line_ended();
2239  _scan_line();
2240  _check_valid_newline_in_quoted_scalar();
2241  }
2242 
2243  _c4err("reached end of file while looking for closing quote");
2244 
2245 found_close:
2246 
2247  _c4dbgpf("found closing quote at: {}", pos);
2248  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, pos != npos, m_evt_handler->m_curr->pos);
2249  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, pos >= 0, m_evt_handler->m_curr->pos);
2250  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.end() >= _buf().begin() && s.end() <= _buf().end(), m_evt_handler->m_curr->pos);
2251  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.end() == _buf().end() || *s.end() == '"', m_evt_handler->m_curr->pos);
2252  _set_first_strict(s, pos);
2253 
2254  _c4prscalar("scanned dquoted scalar", s, /*keep_newlines*/true);
2255 
2256  return ScannedScalar{s, needs_filter};
2257 }
2258 
2259 
2260 //-----------------------------------------------------------------------------
2261 template<class EventHandler>
2262 void ParseEngine<EventHandler>::_scan_block(ScannedBlock *C4_RESTRICT sb, size_t indref)
2263 {
2264  _c4dbgpf("blck: indref={}", indref);
2265  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, indref != npos, m_evt_handler->m_curr->pos);
2266 
2267  // nice explanation here: http://yaml-multiline.info/
2268  csubstr s = m_evt_handler->m_curr->line_contents.rem;
2269  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.begins_with('|') || s.begins_with('>'), m_evt_handler->m_curr->pos);
2270 
2271  _c4dbgpf("blck: specs={}", prs_(s));
2272 
2273  // parse the spec
2274  BlockChomp_e chomp = CHOMP_CLIP; // default to clip unless + or - are used
2275  size_t indentation = npos; // have to find out if no spec is given
2276  if(s.len > 1)
2277  {
2278  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.begins_with_any("|>"), m_evt_handler->m_curr->pos);
2279  csubstr t = s.sub(1);
2280  _c4dbgpf("blck: spec is multichar: {}", prs_(t));
2281  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, t.len >= 1, m_evt_handler->m_curr->pos);
2282  size_t pos = t.first_of("-+");
2283  _c4dbgpf("blck: spec chomp char: pos={}", pos);
2284  if(pos != npos)
2285  {
2286  _c4dbgpf("blck: spec chomp char: {}", _c4prc(t[pos]));
2287  if(t[pos] == '-')
2288  {
2289  _c4dbgp("blck: chomp=STRIP");
2290  chomp = CHOMP_STRIP;
2291  }
2292  else if(t[pos] == '+')
2293  {
2294  _c4dbgp("blck: chomp=KEEP");
2295  chomp = CHOMP_KEEP;
2296  }
2297  if(pos == 0)
2298  t = t.sub(1);
2299  else
2300  t = t.first(pos);
2301  _c4dbgpf("blck: spec is now: {}", prs_(t));
2302  }
2303  // from here to the end, only digits are considered
2304  pos = t.first_not_of("0123456789");
2305  csubstr rest = t.first(pos);
2306  if( ! rest.empty())
2307  {
2308  _c4dbgpf("blck: parse indentation digits: {}", prs_(rest));
2309  if C4_UNLIKELY(rest.len > 1)
2310  _c4err("parse error: invalid indentation");
2311  if C4_UNLIKELY( ! c4::atou(rest, &indentation))
2312  _c4err("parse error: could not read indentation as decimal"); // LCOV_EXCL_LINE
2313  if C4_UNLIKELY( ! indentation)
2314  _c4err("parse error: null indentation");
2315  _c4dbgpf("blck: indentation specified: {}. add {} from curr state -> {}", indentation, m_evt_handler->m_curr->indref, indentation+indref);
2316  indentation += m_evt_handler->m_curr->indref;
2317  }
2318  else
2319  {
2320  rest = t.triml(" \t");
2321  _c4dbgpf("blck: digits empty. t={} trimmed={} iscomm={} t.iscomm={}", prs_(t), prs_(rest), rest.begins_with('#'), t.begins_with('#'));
2322  if C4_UNLIKELY(rest.len && (rest.str[0] != '#' || t.str[0] == '#'))
2323  _c4err("parse error: invalid token");
2324  }
2325  }
2326 
2327  _c4dbgpf("blck: style={} chomp={} indentation={}", s.begins_with('>') ? "fold" : "literal", chomp==CHOMP_CLIP ? "clip" : (chomp==CHOMP_STRIP ? "strip" : "keep"), indentation);
2328 
2329  // finish the current line
2330  _line_progressed(s.len);
2331  _line_ended();
2332  _scan_line();
2333 
2334  // start with a zero-length block, already pointing at the right place
2335  substr raw_block(_buf().data() + m_evt_handler->m_curr->pos.offset, size_t(0));
2336  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, raw_block.begin() == m_evt_handler->m_curr->line_contents.full.str, m_evt_handler->m_curr->pos);
2337 
2338  // read every full line into a raw block,
2339  // from which newlines are to be stripped as needed.
2340  //
2341  // If no explicit indentation was given, pick it from the first
2342  // non-empty line. See
2343  // https://yaml.org/spec/1.2.2/#8111-block-indentation-indicator
2344  size_t num_lines = 0;
2345  size_t first = m_evt_handler->m_curr->pos.line;
2346  size_t provisional_indentation = npos;
2347  LineContents lc;
2348  while(( ! _finished_file()))
2349  {
2350  // peek next line, but do not advance immediately
2351  lc.reset_with_next_line(_buf(), m_evt_handler->m_curr->pos.offset);
2352  #if defined(__GNUC__) && (__GNUC__ == 12 || __GNUC__ == 13)
2353  C4_DONT_OPTIMIZE(lc.rem);
2354  #endif
2355  _c4dbgpf("blck: peeking at {}", prs_(lc.rem.trimr("\r\n"), true));
2356  // evaluate termination conditions
2357  if(indentation != npos)
2358  {
2359  _c4dbgpf("blck: indentation={}", indentation);
2360  // stop when the line is deindented and not empty
2361  if(lc.indentation < indentation && ( ! lc.rem.trim(" \t").empty()))
2362  {
2363  if(raw_block.len)
2364  {
2365  _c4dbgpf("blck: indentation decreased ref={} thisline={}", indentation, lc.indentation);
2366  }
2367  else
2368  {
2369  _c4err("indentation decreased without any scalar");
2370  }
2371  break;
2372  }
2373  else if(indentation == 0)
2374  {
2375  _c4dbgpf("blck: noindent. lc.rem={}", prs_(lc.rem));
2376  if(_is_doc_token(lc.rem))
2377  {
2378  _c4dbgp("blck: stop. indentation=0 and doc ended");
2379  break;
2380  }
2381  }
2382  }
2383  else
2384  {
2385  const size_t fns = lc.rem.first_not_of(' ');
2386  _c4dbgpf("blck: indentation ref not set. firstnonws={}", fns);
2387  if(fns != npos) // non-empty line
2388  {
2389  _c4dbgpf("blck: line not empty. indref={} indprov={} indentation={}", indref, provisional_indentation, lc.indentation);
2390  if C4_UNLIKELY(lc.full.begins_with('\t'))
2391  _c4err("parse error");
2392  if(provisional_indentation == npos)
2393  {
2394  if(lc.indentation < indref)
2395  {
2396  _c4dbgpf("blck: block terminated indentation={} < indref={}", lc.indentation, indref);
2397  if(raw_block.len == 0)
2398  {
2399  _c4dbgp("blck: was empty, undo next line");
2400  _line_ended_undo();
2401  }
2402  break;
2403  }
2404  else if(lc.indentation == m_evt_handler->m_curr->indref)
2405  {
2406  if(has_any(RSEQ|RMAP))
2407  {
2408  _c4dbgpf("blck: block terminated. reading container and indentation={}==indref={}", lc.indentation, m_evt_handler->m_curr->indref);
2409  break;
2410  }
2411  }
2412  _c4dbgpf("blck: set indentation ref from this line: ref={}", lc.indentation);
2413  indentation = lc.indentation;
2414  }
2415  else
2416  {
2417  if(lc.indentation >= provisional_indentation)
2418  {
2419  _c4dbgpf("blck: set indentation ref from provisional indentation: provisional_ref={}, thisline={}", provisional_indentation, lc.indentation);
2420  //indentation = provisional_indentation ? provisional_indentation : lc.indentation;
2421  indentation = lc.indentation;
2422  }
2423  else
2424  {
2425  if(lc.indentation >= indref)
2426  _c4err("parse error: first non-empty block line should have at least the original indentation");
2427  _c4dbgp("blck: finished");
2428  break;
2429  }
2430  }
2431  }
2432  else // empty line
2433  {
2434  _c4dbgpf("blck: line empty or {} spaces. line_indentation={} prov_indentation={}", lc.rem.len, lc.indentation, provisional_indentation);
2435  if(provisional_indentation != npos)
2436  {
2437  if(lc.rem.len >= provisional_indentation)
2438  {
2439  _c4dbgpf("blck: increase provisional_ref {} -> {}", provisional_indentation, lc.rem.len);
2440  provisional_indentation = lc.rem.len;
2441  }
2442  }
2443  else
2444  {
2445  provisional_indentation = lc.indentation ? lc.indentation : has_any(RSEQ|RVAL);
2446  _c4dbgpf("blck: initialize provisional_ref={}", provisional_indentation);
2447  if(provisional_indentation == npos)
2448  {
2449  provisional_indentation = lc.rem.len ? lc.rem.len : has_any(RSEQ|RVAL);
2450  _c4dbgpf("blck: initialize provisional_ref={}", provisional_indentation);
2451  }
2452  if(provisional_indentation < indref)
2453  {
2454  provisional_indentation = indref;
2455  _c4dbgpf("blck: initialize provisional_ref={}", provisional_indentation);
2456  }
2457  }
2458  }
2459  }
2460  // advance now that we know the folded scalar continues
2461  m_evt_handler->m_curr->line_contents = lc;
2462  _c4dbgpf("blck: append '{}'", m_evt_handler->m_curr->line_contents.rem);
2463  raw_block.len += m_evt_handler->m_curr->line_contents.full.len;
2464  _line_progressed(m_evt_handler->m_curr->line_contents.rem.len);
2465  _line_ended();
2466  ++num_lines;
2467  }
2468  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->pos.line == (first + num_lines) || (raw_block.len == 0), m_evt_handler->m_curr->pos);
2469  C4_UNUSED(num_lines);
2470  C4_UNUSED(first);
2471 
2472  if(indentation == npos)
2473  {
2474  _c4dbgpf("blck: set indentation from provisional: {}", provisional_indentation);
2475  indentation = provisional_indentation;
2476  }
2477 
2478  if(num_lines)
2479  _line_ended_undo();
2480 
2481  _c4prscalar("scanned block", raw_block, /*keep_newlines*/true);
2482 
2483  sb->scalar = raw_block;
2484  sb->indentation = indentation;
2485  sb->chomp = chomp;
2486 }
2487 
2488 
2489 //-----------------------------------------------------------------------------
2490 //-----------------------------------------------------------------------------
2491 //-----------------------------------------------------------------------------
2492 /** @cond dev */
2493 
2494 // a debugging scaffold:
2495 #if 0
2496 #define _c4dbgfws(fmt, ...) _c4dbgpf("filt_ws[{}->{}]: " fmt, proc.rpos, proc.wpos, __VA_ARGS__)
2497 #else
2498 #define _c4dbgfws(...)
2499 #endif
2500 
2501 template<class EventHandler>
2502 template<class FilterProcessor>
2503 bool ParseEngine<EventHandler>::_filter_ws_handle_to_first_non_space(FilterProcessor &proc)
2504 {
2505  _c4dbgfws("found whitespace '{}'", _c4prc(proc.curr()));
2506  RYML_ASSERT_PARSE_CB_(this->callbacks(), proc.curr() == ' ' || proc.curr() == '\t', m_evt_handler->m_curr->pos);
2507 
2508  const size_t first_pos = proc.rpos > 0 ? proc.src.first_not_of(" \t", proc.rpos) : proc.src.first_not_of(' ', proc.rpos);
2509  if(first_pos != npos)
2510  {
2511  const char first_char = proc.src[first_pos];
2512  _c4dbgfws("firstnonws='{}'@{}", _c4prc(first_char), first_pos);
2513  if(first_char == '\n' || first_char == '\r') // skip trailing whitespace
2514  {
2515  _c4dbgfws("whitespace is trailing on line", "");
2516  proc.skip(first_pos - proc.rpos);
2517  }
2518  else // a legit whitespace
2519  {
2520  proc.copy();
2521  _c4dbgfws("legit whitespace. sofar={}", prs_(proc.sofar()));
2522  }
2523  return true;
2524  }
2525  _c4dbgfws("whitespace is trailing on line", "");
2526  return false;
2527 }
2528 
2529 template<class EventHandler>
2530 template<class FilterProcessor>
2531 void ParseEngine<EventHandler>::_filter_ws_copy_trailing(FilterProcessor &proc)
2532 {
2533  if(!_filter_ws_handle_to_first_non_space(proc))
2534  {
2535  _c4dbgfws("... everything else is trailing whitespace - copy {} chars", proc.src.len - proc.rpos);
2536  proc.copy(proc.src.len - proc.rpos);
2537  }
2538 }
2539 
2540 template<class EventHandler>
2541 template<class FilterProcessor>
2542 void ParseEngine<EventHandler>::_filter_ws_skip_trailing(FilterProcessor &proc)
2543 {
2544  if(!_filter_ws_handle_to_first_non_space(proc))
2545  {
2546  _c4dbgfws("... everything else is trailing whitespace - skip {} chars", proc.src.len - proc.rpos);
2547  proc.skip(proc.src.len - proc.rpos);
2548  }
2549 }
2550 
2551 #undef _c4dbgfws
2552 
2553 
2554 //-----------------------------------------------------------------------------
2555 //-----------------------------------------------------------------------------
2556 //-----------------------------------------------------------------------------
2557 /* plain scalars */
2558 
2559 // a debugging scaffold:
2560 #if 0
2561 #define _c4dbgfps(fmt, ...) _c4dbgpf("filt_plain[{}->{}]: " fmt, proc.rpos, proc.wpos, __VA_ARGS__)
2562 #else
2563 #define _c4dbgfps(fmt, ...)
2564 #endif
2565 
2566 template<class EventHandler>
2567 template<class FilterProcessor>
2568 void ParseEngine<EventHandler>::_filter_nl_plain(FilterProcessor &C4_RESTRICT proc, size_t indentation)
2569 {
2570  RYML_ASSERT_PARSE_CB_(this->callbacks(), proc.curr() == '\n', m_evt_handler->m_curr->pos);
2571 
2572  _c4dbgfps("found newline. sofar={}", prs_(proc.sofar()));
2573  size_t ii = proc.rpos;
2574  const size_t numnl_following = _count_following_newlines(proc.src, &ii, indentation);
2575  if(numnl_following)
2576  {
2577  proc.set('\n', numnl_following);
2578  _c4dbgfps("{} consecutive (empty) lines {}. totalws={}", 1+numnl_following, ii < proc.src.len ? "in the middle" : "at the end", proc.rpos-ii);
2579  }
2580  else
2581  {
2582  const size_t ret = proc.src.first_not_of(" \t", proc.rpos+1);
2583  if(ret != npos)
2584  {
2585  proc.set(' ');
2586  _c4dbgfps("single newline. convert to space. ret={}/{}. sofar={}", ii, proc.src.len, prs_(proc.sofar()));
2587  }
2588  else
2589  {
2590  _c4dbgfps("last newline, everything else is whitespace. ii={}/{}", ii, proc.src.len);
2591  ii = proc.src.len;
2592  }
2593  }
2594  proc.rpos = ii;
2595 }
2596 
2597 template<class EventHandler>
2598 template<class FilterProcessor>
2599 auto ParseEngine<EventHandler>::_filter_plain(FilterProcessor &C4_RESTRICT proc, size_t indentation) -> decltype(proc.result())
2600 {
2601  RYML_ASSERT_PARSE_CB_(this->callbacks(), indentation != npos, m_evt_handler->m_curr->pos);
2602  _c4dbgfps("before={}", prs_(proc.src));
2603 
2604  while(proc.has_more_chars())
2605  {
2606  const char curr = proc.curr();
2607  _c4dbgfps("'{}', sofar={}", _c4prc(curr), prs_(proc.sofar()));
2608  switch(curr)
2609  {
2610  case ' ':
2611  RYML_WITH_TAB_TOKENS_(case '\t':)
2612  _c4dbgfps("whitespace", curr);
2613  _filter_ws_skip_trailing(proc);
2614  break;
2615  case '\n':
2616  _c4dbgfps("newline", curr);
2617  _filter_nl_plain(proc, /*indentation*/indentation);
2618  break;
2619  case '\r': // skip \r --- https://stackoverflow.com/questions/1885900
2620  _c4dbgfps("carriage return, ignore", curr);
2621  proc.skip();
2622  break;
2623  default:
2624  proc.copy();
2625  break;
2626  }
2627  }
2628 
2629  _c4dbgfps("after={}", prs_(proc.sofar()));
2630 
2631  return proc.result();
2632 }
2633 
2634 #undef _c4dbgfps
2635 
2636 
2637 template<class EventHandler>
2638 FilterResult ParseEngine<EventHandler>::filter_scalar_plain(csubstr scalar, substr dst, size_t indentation)
2639 {
2640  FilterProcessorSrcDst proc(scalar, dst);
2641  return _filter_plain(proc, indentation);
2642 }
2643 
2644 template<class EventHandler>
2645 FilterResult ParseEngine<EventHandler>::filter_scalar_plain_in_place(substr dst, size_t cap, size_t indentation)
2646 {
2647  FilterProcessorInplaceEndExtending proc(dst, cap);
2648  return _filter_plain(proc, indentation);
2649 }
2650 
2651 
2652 //-----------------------------------------------------------------------------
2653 //-----------------------------------------------------------------------------
2654 //-----------------------------------------------------------------------------
2655 /* single quoted */
2656 
2657 // a debugging scaffold:
2658 #if 0
2659 #define _c4dbgfsq(fmt, ...) _c4dbgpf("filt_squo[{}->{}]: " fmt, proc.rpos, proc.wpos, __VA_ARGS__)
2660 #else
2661 #define _c4dbgfsq(fmt, ...)
2662 #endif
2663 
2664 template<class EventHandler>
2665 template<class FilterProcessor>
2666 void ParseEngine<EventHandler>::_filter_nl_squoted(FilterProcessor &C4_RESTRICT proc)
2667 {
2668  RYML_ASSERT_PARSE_CB_(this->callbacks(), proc.curr() == '\n', m_evt_handler->m_curr->pos);
2669 
2670  _c4dbgfsq("found newline. sofar={}", prs_(proc.sofar()));
2671  size_t ii = proc.rpos;
2672  const size_t numnl_following = _count_following_newlines(proc.src, &ii);
2673  if(numnl_following)
2674  {
2675  proc.set('\n', numnl_following);
2676  _c4dbgfsq("{} consecutive (empty) lines {}. totalws={}", 1+numnl_following, ii < proc.src.len ? "in the middle" : "at the end", proc.rpos-ii);
2677  }
2678  else
2679  {
2680  const size_t ret = proc.src.first_not_of(" \t", proc.rpos+1);
2681  if(ret != npos)
2682  {
2683  proc.set(' ');
2684  _c4dbgfsq("single newline. convert to space. ret={}/{}. sofar={}", ii, proc.src.len, prs_(proc.sofar()));
2685  }
2686  else
2687  {
2688  proc.set(' ');
2689  _c4dbgfsq("single newline. convert to space. ii={}/{}. sofar={}", ii, proc.src.len, prs_(proc.sofar()));
2690  }
2691  }
2692  proc.rpos = ii;
2693 }
2694 
2695 template<class EventHandler>
2696 template<class FilterProcessor>
2697 auto ParseEngine<EventHandler>::_filter_squoted(FilterProcessor &C4_RESTRICT proc) -> decltype(proc.result())
2698 {
2699  _c4dbgfsq("before={}", prs_(proc.src));
2700 
2701  // from the YAML spec for double-quoted scalars:
2702  // https://yaml.org/spec/1.2-old/spec.html#style/flow/single-quoted
2703  while(proc.has_more_chars())
2704  {
2705  const char curr = proc.curr();
2706  _c4dbgfsq("'{}', sofar={}", _c4prc(curr), prs_(proc.sofar()));
2707  switch(curr)
2708  {
2709  case ' ':
2710  case '\t':
2711  _c4dbgfsq("whitespace", curr);
2712  _filter_ws_copy_trailing(proc);
2713  break;
2714  case '\n':
2715  _c4dbgfsq("newline", curr);
2716  _filter_nl_squoted(proc);
2717  break;
2718  case '\r': // skip \r --- https://stackoverflow.com/questions/1885900
2719  _c4dbgfsq("skip cr", curr);
2720  proc.skip();
2721  break;
2722  case '\'':
2723  _c4dbgfsq("squote", curr);
2724  if(proc.next() == '\'')
2725  {
2726  _c4dbgfsq("two consecutive squotes", curr);
2727  proc.skip();
2728  proc.copy();
2729  }
2730  else
2731  {
2732  _c4err("filter error");
2733  }
2734  break;
2735  default:
2736  proc.copy();
2737  break;
2738  }
2739  }
2740 
2741  _c4dbgfsq(": #filteredchars={} after={}", proc.src.len-proc.sofar().len, prs_(proc.sofar()));
2742 
2743  return proc.result();
2744 }
2745 
2746 #undef _c4dbgfsq
2747 
2748 template<class EventHandler>
2749 FilterResult ParseEngine<EventHandler>::filter_scalar_squoted(csubstr scalar, substr dst)
2750 {
2751  FilterProcessorSrcDst proc(scalar, dst);
2752  return _filter_squoted(proc);
2753 }
2754 
2755 template<class EventHandler>
2756 FilterResult ParseEngine<EventHandler>::filter_scalar_squoted_in_place(substr dst, size_t cap)
2757 {
2758  FilterProcessorInplaceEndExtending proc(dst, cap);
2759  return _filter_squoted(proc);
2760 }
2761 
2762 
2763 //-----------------------------------------------------------------------------
2764 //-----------------------------------------------------------------------------
2765 //-----------------------------------------------------------------------------
2766 /* double quoted */
2767 
2768 // a debugging scaffold:
2769 #if 0
2770 #define _c4dbgfdq(fmt, ...) _c4dbgpf("filt_dquo[{}->{}]: " fmt, proc.rpos, proc.wpos, __VA_ARGS__)
2771 #else
2772 #define _c4dbgfdq(...)
2773 #endif
2774 
2775 template<class EventHandler>
2776 template<class FilterProcessor>
2777 void ParseEngine<EventHandler>::_filter_nl_dquoted(FilterProcessor &C4_RESTRICT proc)
2778 {
2779  RYML_ASSERT_PARSE_CB_(this->callbacks(), proc.curr() == '\n', m_evt_handler->m_curr->pos);
2780 
2781  _c4dbgfdq("found newline. sofar={}", prs_(proc.sofar()));
2782  size_t ii = proc.rpos;
2783  const size_t numnl_following = _count_following_newlines(proc.src, &ii);
2784  if(numnl_following)
2785  {
2786  proc.set('\n', numnl_following);
2787  _c4dbgfdq("{} consecutive (empty) lines {}. totalws={}", 1+numnl_following, ii < proc.src.len ? "in the middle" : "at the end", proc.rpos-ii);
2788  }
2789  else
2790  {
2791  const size_t ret = proc.src.first_not_of(" \t", proc.rpos+1);
2792  if(ret != npos)
2793  {
2794  proc.set(' ');
2795  _c4dbgfdq("single newline. convert to space. ret={}/{}. sofar={}", ii, proc.src.len, prs_(proc.sofar()));
2796  }
2797  else
2798  {
2799  proc.set(' ');
2800  _c4dbgfdq("single newline. convert to space. ii={}/{}. sofar={}", ii, proc.src.len, prs_(proc.sofar()));
2801  }
2802  if(ii < proc.src.len && proc.src.str[ii] == '\\')
2803  {
2804  _c4dbgfdq("backslash at [{}]", ii);
2805  const char next = ii+1 < proc.src.len ? proc.src.str[ii+1] : '\0';
2806  if(next == ' ' || next == '\t')
2807  {
2808  _c4dbgfdq("extend skip to backslash", "");
2809  ++ii;
2810  }
2811  }
2812  }
2813  proc.rpos = ii;
2814 }
2815 
2816 template<class EventHandler>
2817 template<class FilterProcessor>
2818 void ParseEngine<EventHandler>::_filter_dquoted_backslash_decode(FilterProcessor &C4_RESTRICT proc, size_t sz)
2819 {
2820  const size_t szp1 = sz + 1u;
2821  if C4_UNLIKELY(proc.rpos + szp1 >= proc.src.len)
2822  _c4err("codepoint requires {} hex digits. scalar pos={}", sz, proc.rpos);
2823  char readbuf[8];
2824  csubstr codepoint = proc.src.sub(proc.rpos + 2u, sz);
2825  _c4dbgfdq("utf8 ~~~{}~~~ rpos={} rem=~~~{}~~~", codepoint, proc.rpos, proc.src.sub(proc.rpos));
2826  uint32_t codepoint_val = {};
2827  if C4_UNLIKELY(!read_hex(codepoint, &codepoint_val))
2828  _c4err("failed to parse codepoint. scalar pos={}", proc.rpos);
2829  const size_t numbytes = decode_code_point((uint8_t*)readbuf, sizeof(readbuf), codepoint_val);
2830  if C4_UNLIKELY(numbytes == 0)
2831  _c4err("failed to decode code point={}", proc.rpos);
2832  RYML_ASSERT_PARSE_CB_(callbacks(), numbytes <= 4, m_evt_handler->m_curr->pos);
2833  proc.translate_esc_bulk(readbuf, numbytes, /*nread*/szp1);
2834  _c4dbgfdq("utf8 after rpos={} rem=~~~{}~~~", proc.rpos, proc.src.sub(proc.rpos));
2835 }
2836 
2837 template<class EventHandler>
2838 template<class FilterProcessor>
2839 void ParseEngine<EventHandler>::_filter_dquoted_backslash(FilterProcessor &C4_RESTRICT proc)
2840 {
2841  char next = proc.next();
2842  _c4dbgfdq("backslash, next='{}'", _c4prc(next));
2843  if(next == '\r')
2844  {
2845  if(proc.rpos+2 < proc.src.len && proc.src.str[proc.rpos+2] == '\n')
2846  {
2847  proc.skip(); // newline escaped with \ -- skip both (add only one as i is loop-incremented)
2848  next = '\n';
2849  _c4dbgfdq("[{}]: was \\r\\n, now next='\\n'", proc.rpos);
2850  }
2851  }
2852 
2853  if(next == '\n')
2854  {
2855  size_t ii = proc.rpos + 2;
2856  for( ; ii < proc.src.len; ++ii)
2857  {
2858  // skip leading whitespace
2859  if(proc.src.str[ii] == ' ' || proc.src.str[ii] == '\t')
2860  ;
2861  else
2862  break;
2863  }
2864  proc.skip(ii - proc.rpos);
2865  }
2866  else if(next == '"' || next == '/' || next == ' ' || next == '\t')
2867  {
2868  // escapes for json compatibility
2869  proc.translate_esc(next);
2870  _c4dbgfdq("here, used '{}'", _c4prc(next));
2871  }
2872  else if(next == '\r')
2873  {
2874  proc.skip();
2875  }
2876  else if(next == 'n')
2877  {
2878  proc.translate_esc('\n');
2879  }
2880  else if(next == 'r')
2881  {
2882  proc.translate_esc('\r');
2883  }
2884  else if(next == 't')
2885  {
2886  proc.translate_esc('\t');
2887  }
2888  else if(next == '\\')
2889  {
2890  proc.translate_esc('\\');
2891  }
2892  else if(next == 'x') // 2-digit Unicode escape (\xXX), code point 0x00–0xFF
2893  {
2894  _filter_dquoted_backslash_decode(proc, 2u);
2895  }
2896  else if(next == 'u') // 4-digit Unicode escape (\uXXXX), code point 0x0000–0xFFFF
2897  {
2898  _filter_dquoted_backslash_decode(proc, 4u);
2899  }
2900  else if(next == 'U') // 8-digit Unicode escape (\UXXXXXXXX), full 32-bit code point
2901  {
2902  _filter_dquoted_backslash_decode(proc, 8u);
2903  }
2904  // https://yaml.org/spec/1.2.2/#rule-c-ns-esc-char
2905  else if(next == '0')
2906  {
2907  proc.translate_esc('\0');
2908  }
2909  else if(next == 'b') // backspace
2910  {
2911  proc.translate_esc('\b');
2912  }
2913  else if(next == 'f') // form feed
2914  {
2915  proc.translate_esc('\f');
2916  }
2917  else if(next == 'a') // bell character
2918  {
2919  proc.translate_esc('\a');
2920  }
2921  else if(next == 'v') // vertical tab
2922  {
2923  proc.translate_esc('\v');
2924  }
2925  else if(next == 'e') // escape character
2926  {
2927  proc.translate_esc('\x1b');
2928  }
2929  else if(next == '_') // unicode non breaking space \u00a0
2930  {
2931  // https://www.compart.com/en/unicode/U+00a0
2932  const char payload[] = {
2933  RYML_CHCONST_(-0x3e, 0xc2),
2934  RYML_CHCONST_(-0x60, 0xa0),
2935  };
2936  proc.translate_esc_bulk(payload, /*nwrite*/2, /*nread*/1);
2937  }
2938  else if(next == 'N') // unicode next line \u0085
2939  {
2940  // https://www.compart.com/en/unicode/U+0085
2941  const char payload[] = {
2942  RYML_CHCONST_(-0x3e, 0xc2),
2943  RYML_CHCONST_(-0x7b, 0x85),
2944  };
2945  proc.translate_esc_bulk(payload, /*nwrite*/2, /*nread*/1);
2946  }
2947  else if(next == 'L') // unicode line separator \u2028
2948  {
2949  // https://www.utf8-chartable.de/unicode-utf8-table.pl?start=8192&number=1024&names=-&utf8=0x&unicodeinhtml=hex
2950  const char payload[] = {
2951  RYML_CHCONST_(-0x1e, 0xe2),
2952  RYML_CHCONST_(-0x80, 0x80),
2953  RYML_CHCONST_(-0x58, 0xa8),
2954  };
2955  proc.translate_esc_extending(payload, /*nwrite*/3, /*nread*/1);
2956  }
2957  else if(next == 'P') // unicode paragraph separator \u2029
2958  {
2959  // https://www.utf8-chartable.de/unicode-utf8-table.pl?start=8192&number=1024&names=-&utf8=0x&unicodeinhtml=hex
2960  const char payload[] = {
2961  RYML_CHCONST_(-0x1e, 0xe2),
2962  RYML_CHCONST_(-0x80, 0x80),
2963  RYML_CHCONST_(-0x57, 0xa9),
2964  };
2965  proc.translate_esc_extending(payload, /*nwrite*/3, /*nread*/1);
2966  }
2967  else if(next == '\0')
2968  {
2969  proc.skip();
2970  }
2971  else
2972  {
2973  _c4err("unknown character '{}' after '\\' pos={}", _c4prc(next), proc.rpos);
2974  }
2975  _c4dbgfdq("backslash...sofar={}", prs_(proc.sofar()));
2976 }
2977 
2978 
2979 template<class EventHandler>
2980 template<class FilterProcessor>
2981 auto ParseEngine<EventHandler>::_filter_dquoted(FilterProcessor &C4_RESTRICT proc) -> decltype(proc.result())
2982 {
2983  _c4dbgfdq("before={}", prs_(proc.src));
2984  // from the YAML spec for double-quoted scalars:
2985  // https://yaml.org/spec/1.2-old/spec.html#style/flow/double-quoted
2986  while(proc.has_more_chars())
2987  {
2988  const char curr = proc.curr();
2989  _c4dbgfdq("'{}' sofar={}", _c4prc(curr), prs_(proc.sofar()));
2990  switch(curr)
2991  {
2992  case ' ':
2993  case '\t':
2994  {
2995  _c4dbgfdq("whitespace", curr);
2996  _filter_ws_copy_trailing(proc);
2997  break;
2998  }
2999  case '\n':
3000  {
3001  _c4dbgfdq("newline", curr);
3002  _filter_nl_dquoted(proc);
3003  break;
3004  }
3005  case '\r': // skip \r --- https://stackoverflow.com/questions/1885900
3006  {
3007  _c4dbgfdq("carriage return, ignore", curr);
3008  proc.skip();
3009  break;
3010  }
3011  case '\\':
3012  {
3013  _filter_dquoted_backslash(proc);
3014  break;
3015  }
3016  default:
3017  {
3018  proc.copy();
3019  break;
3020  }
3021  }
3022  }
3023  _c4dbgfdq("after={}", prs_(proc.sofar()));
3024  return proc.result();
3025 }
3026 
3027 #undef _c4dbgfdq
3028 
3029 
3030 template<class EventHandler>
3031 FilterResult ParseEngine<EventHandler>::filter_scalar_dquoted(csubstr scalar, substr dst)
3032 {
3033  FilterProcessorSrcDst proc(scalar, dst);
3034  return _filter_dquoted(proc);
3035 }
3036 
3037 template<class EventHandler>
3038 FilterResultExtending ParseEngine<EventHandler>::filter_scalar_dquoted_in_place(substr dst, size_t cap)
3039 {
3040  FilterProcessorInplaceMidExtending proc(dst, cap);
3041  return _filter_dquoted(proc);
3042 }
3043 
3044 
3045 //-----------------------------------------------------------------------------
3046 //-----------------------------------------------------------------------------
3047 //-----------------------------------------------------------------------------
3048 // block filtering helpers
3049 
3050 template<class EventHandler>
3051 template<class FilterProcessor>
3052 void ParseEngine<EventHandler>::_filter_chomp(FilterProcessor &C4_RESTRICT proc, BlockChomp_e chomp, size_t indentation)
3053 {
3054  RYML_ASSERT_PARSE_CB_(this->callbacks(), chomp == CHOMP_CLIP || chomp == CHOMP_KEEP || chomp == CHOMP_STRIP, m_evt_handler->m_curr->pos);
3055  RYML_ASSERT_PARSE_CB_(this->callbacks(), proc.rem().first_not_of(" \n\r") == npos, m_evt_handler->m_curr->pos);
3056 
3057  // a debugging scaffold:
3058  #if 0
3059  #define _c4dbgchomp(fmt, ...) _c4dbgpf("chomp[{}->{}]: " fmt, proc.rpos, proc.wpos, __VA_ARGS__)
3060  #else
3061  #define _c4dbgchomp(...)
3062  #endif
3063 
3064  // advance to the last line having spaces beyond the indentation
3065  {
3066  size_t last = _find_last_newline_and_larger_indentation(proc.rem(), indentation);
3067  if(last != npos)
3068  {
3069  _c4dbgchomp("found newline and larger indentation. last={}", last);
3070  last = proc.rpos + last + size_t(1) + indentation; // last started at to-be-read.
3071  RYML_ASSERT_PARSE_CB_(this->callbacks(), last <= proc.src.len, m_evt_handler->m_curr->pos);
3072  // remove indentation spaces, copy the rest
3073  while((proc.rpos < last) && proc.has_more_chars())
3074  {
3075  const char curr = proc.curr();
3076  _c4dbgchomp("curr='{}'", _c4prc(curr));
3077  switch(curr)
3078  {
3079  case '\n':
3080  {
3081  _c4dbgchomp("newline! remlen={}", proc.rem().len);
3082  proc.copy();
3083  // are there spaces after the newline?
3084  csubstr at_next_line = proc.rem();
3085  if(at_next_line.begins_with(' '))
3086  {
3087  _c4dbgchomp("next line begins with spaces. indentation={}", indentation);
3088  // there are spaces.
3089  size_t first_non_space = at_next_line.first_not_of(' ');
3090  _c4dbgchomp("first_non_space={}", first_non_space);
3091  if(first_non_space == npos)
3092  {
3093  _c4dbgchomp("{} spaces, to the end", at_next_line.len);
3094  first_non_space = at_next_line.len;
3095  }
3096  if(first_non_space <= indentation)
3097  {
3098  _c4dbgchomp("skip spaces={}<=indentation={}", first_non_space, indentation);
3099  proc.skip(first_non_space);
3100  }
3101  else
3102  {
3103  _c4dbgchomp("skip indentation={}<spaces={}", indentation, first_non_space);
3104  proc.skip(indentation);
3105  // copy the spaces after the indentation
3106  _c4dbgchomp("copy {}={}-{} spaces", first_non_space - indentation, first_non_space, indentation);
3107  proc.copy(first_non_space - indentation);
3108  }
3109  }
3110  break;
3111  }
3112  case '\r':
3113  proc.skip();
3114  break;
3115  }
3116  }
3117  }
3118  }
3119 
3120  // from now on, we only have line ends (or indentation spaces)
3121  switch(chomp)
3122  {
3123  case CHOMP_CLIP:
3124  {
3125  bool had_one = false;
3126  while(proc.has_more_chars())
3127  {
3128  const char curr = proc.curr();
3129  _c4dbgchomp("CLIP: '{}'", _c4prc(curr));
3130  switch(curr)
3131  {
3132  case '\n':
3133  {
3134  _c4dbgchomp("copy newline!", curr);
3135  proc.copy();
3136  proc.set_at_end();
3137  had_one = true;
3138  break;
3139  }
3140  case ' ':
3141  case '\r':
3142  _c4dbgchomp("skip!", curr);
3143  proc.skip();
3144  break;
3145  }
3146  }
3147  if(!had_one) // there were no newline characters. add one.
3148  {
3149  _c4dbgchomp("chomp=CLIP: add missing newline @{}", proc.wpos);
3150  proc.set('\n');
3151  }
3152  break;
3153  }
3154  case CHOMP_KEEP:
3155  {
3156  _c4dbgchomp("chomp=KEEP: copy all remaining new lines of {} characters", proc.rem().len);
3157  while(proc.has_more_chars())
3158  {
3159  const char curr = proc.curr();
3160  _c4dbgchomp("KEEP: '{}'", _c4prc(curr));
3161  switch(curr)
3162  {
3163  case '\n':
3164  _c4dbgchomp("copy newline!", curr);
3165  proc.copy();
3166  break;
3167  case ' ':
3168  case '\r':
3169  _c4dbgchomp("skip!", curr);
3170  proc.skip();
3171  break;
3172  }
3173  }
3174  break;
3175  }
3176  case CHOMP_STRIP:
3177  {
3178  _c4dbgchomp("chomp=STRIP: strip {} characters", proc.rem().len);
3179  // nothing to do!
3180  break;
3181  }
3182  }
3183 
3184  #undef _c4dbgchomp
3185 }
3186 
3187 
3188 // a debugging scaffold:
3189 #if 0
3190 #define _c4dbgfb(fmt, ...) _c4dbgpf("filt_block[{}->{}]: " fmt, proc.rpos, proc.wpos, __VA_ARGS__)
3191 #else
3192 #define _c4dbgfb(...)
3193 #endif
3194 
3195 template<class EventHandler>
3196 template<class FilterProcessor>
3197 void ParseEngine<EventHandler>::_filter_block_indentation(FilterProcessor &C4_RESTRICT proc, size_t indentation)
3198 {
3199  csubstr rem = proc.rem(); // remaining
3200  if(rem.len)
3201  {
3202  size_t first = rem.first_not_of(' ');
3203  if(first != npos)
3204  {
3205  _c4dbgfb("{} spaces follow before next nonws character", first);
3206  if(first < indentation)
3207  {
3208  _c4dbgfb("skip {}<{} spaces from indentation", first, indentation);
3209  proc.skip(first);
3210  }
3211  else
3212  {
3213  _c4dbgfb("skip {} spaces from indentation", indentation);
3214  proc.skip(indentation);
3215  }
3216  }
3217  #ifdef RYML_NO_COVERAGE__TO_BE_DELETED
3218  else
3219  {
3220  _c4dbgfb("all spaces to the end: {} spaces", first);
3221  first = rem.len;
3222  if(first)
3223  {
3224  if(first < indentation)
3225  {
3226  _c4dbgfb("skip everything", first);
3227  proc.skip(proc.src.len - proc.rpos);
3228  }
3229  else
3230  {
3231  _c4dbgfb("skip {} spaces from indentation", indentation);
3232  proc.skip(indentation);
3233  }
3234  }
3235  }
3236  #endif
3237  }
3238 }
3239 
3240 template<class EventHandler>
3241 template<class FilterProcessor>
3242 size_t ParseEngine<EventHandler>::_handle_all_whitespace(FilterProcessor &C4_RESTRICT proc, BlockChomp_e chomp)
3243 {
3244  csubstr contents = proc.src.trimr(" \n\r");
3245  _c4dbgfb("ws: contents_len={} wslen={}", contents.len, proc.src.len-contents.len);
3246  if(!contents.len)
3247  {
3248  _c4dbgfb("ws: all whitespace: len={}", proc.src.len);
3249  if(chomp == CHOMP_KEEP && proc.src.len)
3250  {
3251  _c4dbgfb("ws: chomp=KEEP all {} newlines", proc.src.count('\n'));
3252  while(proc.has_more_chars())
3253  {
3254  const char curr = proc.curr();
3255  if(curr == '\n')
3256  proc.copy();
3257  else
3258  proc.skip();
3259  }
3260  if(!proc.wpos)
3261  {
3262  proc.set('\n');
3263  }
3264  }
3265  }
3266  return contents.len;
3267 }
3268 
3269 template<class EventHandler>
3270 template<class FilterProcessor>
3271 size_t ParseEngine<EventHandler>::_extend_to_chomp(FilterProcessor &C4_RESTRICT proc, size_t contents_len)
3272 {
3273  _c4dbgfb("contents_len={}", contents_len);
3274 
3275  RYML_ASSERT_PARSE_CB_(this->callbacks(), contents_len > 0u, m_evt_handler->m_curr->pos);
3276 
3277  // extend contents to just before the first newline at the end,
3278  // in case it is preceded by spaces
3279  size_t firstnewl = proc.src.first_of('\n', contents_len);
3280  if(firstnewl != npos)
3281  {
3282  contents_len = firstnewl;
3283  _c4dbgfb("contents_len={} <--- firstnewl={}", contents_len, firstnewl);
3284  }
3285  else
3286  {
3287  contents_len = proc.src.len;
3288  _c4dbgfb("contents_len={} <--- src.len={}", contents_len, proc.src.len);
3289  }
3290 
3291  return contents_len;
3292 }
3293 
3294 #undef _c4dbgfb
3295 
3296 
3297 //-----------------------------------------------------------------------------
3298 //-----------------------------------------------------------------------------
3299 //-----------------------------------------------------------------------------
3300 
3301 // a debugging scaffold:
3302 #if 0
3303 #define _c4dbgfbl(fmt, ...) _c4dbgpf("filt_block_lit[{}->{}]: " fmt, proc.rpos, proc.wpos, __VA_ARGS__)
3304 #else
3305 #define _c4dbgfbl(...)
3306 #endif
3307 
3308 template<class EventHandler>
3309 template<class FilterProcessor>
3310 auto ParseEngine<EventHandler>::_filter_block_literal(FilterProcessor &C4_RESTRICT proc, size_t indentation, BlockChomp_e chomp) -> decltype(proc.result())
3311 {
3312  _c4dbgfbl("indentation={} before={}", indentation, prs_(proc.src));
3313 
3314  size_t contents_len = _handle_all_whitespace(proc, chomp);
3315  if(!contents_len)
3316  return proc.result();
3317 
3318  contents_len = _extend_to_chomp(proc, contents_len);
3319 
3320  _c4dbgfbl("to filter={}", prs_(proc.src.first(contents_len)));
3321 
3322  _filter_block_indentation(proc, indentation);
3323 
3324  // now filter the bulk
3325  while(proc.has_more_chars(/*maxpos*/contents_len))
3326  {
3327  const char curr = proc.curr();
3328  _c4dbgfbl("'{}' sofar={}", _c4prc(curr), prs_(proc.sofar()));
3329  switch(curr)
3330  {
3331  case '\n':
3332  {
3333  _c4dbgfbl("found newline. skip indentation on the next line", curr);
3334  proc.copy(); // copy the newline
3335  _filter_block_indentation(proc, indentation);
3336  break;
3337  }
3338  case '\r':
3339  proc.skip();
3340  break;
3341  default:
3342  proc.copy();
3343  break;
3344  }
3345  }
3346 
3347  _c4dbgfbl("before chomp: #tochomp={} sofar={}", proc.rem().len, prs_(proc.sofar()));
3348 
3349  _filter_chomp(proc, chomp, indentation);
3350 
3351  _c4dbgfbl("final={}", prs_(proc.sofar()));
3352 
3353  return proc.result();
3354 }
3355 
3356 #undef _c4dbgfbl
3357 
3358 template<class EventHandler>
3359 FilterResult ParseEngine<EventHandler>::filter_scalar_block_literal(csubstr scalar, substr dst, size_t indentation, BlockChomp_e chomp)
3360 {
3361  FilterProcessorSrcDst proc(scalar, dst);
3362  return _filter_block_literal(proc, indentation, chomp);
3363 }
3364 
3365 template<class EventHandler>
3366 FilterResult ParseEngine<EventHandler>::filter_scalar_block_literal_in_place(substr scalar, size_t cap, size_t indentation, BlockChomp_e chomp)
3367 {
3368  FilterProcessorInplaceEndExtending proc(scalar, cap);
3369  return _filter_block_literal(proc, indentation, chomp);
3370 }
3371 
3372 
3373 //-----------------------------------------------------------------------------
3374 //-----------------------------------------------------------------------------
3375 //-----------------------------------------------------------------------------
3376 
3377 // a debugging scaffold:
3378 #if 0
3379 #define _c4dbgfbf(fmt, ...) _c4dbgpf("filt_block_folded[{}->{}]: " fmt, proc.rpos, proc.wpos, __VA_ARGS__)
3380 #else
3381 #define _c4dbgfbf(...)
3382 #endif
3383 
3384 
3385 template<class EventHandler>
3386 template<class FilterProcessor>
3387 void ParseEngine<EventHandler>::_filter_block_folded_newlines_leading(FilterProcessor &C4_RESTRICT proc, size_t indentation, size_t len)
3388 {
3389  _filter_block_indentation(proc, indentation);
3390  while(proc.has_more_chars(len))
3391  {
3392  const char curr = proc.curr();
3393  _c4dbgfbf("'{}' sofar={}", _c4prc(curr), prs_(proc.sofar()));
3394  switch(curr)
3395  {
3396  case '\n':
3397  _c4dbgfbf("newline.", curr);
3398  proc.copy();
3399  _filter_block_indentation(proc, indentation);
3400  break;
3401  case '\r':
3402  proc.skip();
3403  break;
3404  case ' ':
3405  case '\t':
3406  {
3407  size_t first = proc.rem().first_not_of(" \t");
3408  _c4dbgfbf("space. first={}", first);
3409  if(first == npos)
3410  first = proc.rem().len;
3411  _c4dbgfbf("... indentation increased to {}", first);
3412  _filter_block_folded_indented_block(proc, indentation, len, first);
3413  break;
3414  }
3415  default:
3416  _c4dbgfbf("newl leading: not space, not newline. stop.", 0);
3417  return;
3418  }
3419  }
3420 }
3421 
3422 template<class EventHandler>
3423 template<class FilterProcessor>
3424 size_t ParseEngine<EventHandler>::_filter_block_folded_newlines_compress(FilterProcessor &C4_RESTRICT proc, size_t num_newl, size_t wpos_at_first_newl)
3425 {
3426  switch(num_newl)
3427  {
3428  case 1u:
3429  _c4dbgfbf("... this is the first newline. turn into space. wpos={}", proc.wpos);
3430  wpos_at_first_newl = proc.wpos;
3431  proc.skip();
3432  proc.set(' ');
3433  break;
3434  case 2u:
3435  _c4dbgfbf("... this is the second newline. prev space (at wpos={}) must be newline", wpos_at_first_newl);
3436  RYML_ASSERT_PARSE_CB_(this->callbacks(), wpos_at_first_newl != npos, m_evt_handler->m_curr->pos);
3437  RYML_ASSERT_PARSE_CB_(this->callbacks(), proc.sofar()[wpos_at_first_newl] == ' ', m_evt_handler->m_curr->pos);
3438  RYML_ASSERT_PARSE_CB_(this->callbacks(), wpos_at_first_newl + 1u == proc.wpos, m_evt_handler->m_curr->pos);
3439  proc.skip();
3440  proc.set_at(wpos_at_first_newl, '\n');
3441  RYML_ASSERT_PARSE_CB_(this->callbacks(), proc.sofar()[wpos_at_first_newl] == '\n', m_evt_handler->m_curr->pos);
3442  break;
3443  default:
3444  _c4dbgfbf("... subsequent newline (num_newl={}). copy", num_newl);
3445  proc.copy();
3446  break;
3447  }
3448  return wpos_at_first_newl;
3449 }
3450 
3451 template<class EventHandler>
3452 template<class FilterProcessor>
3453 void ParseEngine<EventHandler>::_filter_block_folded_newlines(FilterProcessor &C4_RESTRICT proc, size_t indentation, size_t len)
3454 {
3455  RYML_ASSERT_PARSE_CB_(this->callbacks(), proc.curr() == '\n', m_evt_handler->m_curr->pos);
3456  size_t num_newl = 0;
3457  size_t wpos_at_first_newl = npos;
3458  while(proc.has_more_chars(len))
3459  {
3460  const char curr = proc.curr();
3461  _c4dbgfbf("'{}' sofar={}", _c4prc(curr), prs_(proc.sofar()));
3462  switch(curr)
3463  {
3464  case '\n':
3465  {
3466  _c4dbgfbf("newline. sofar={}", num_newl);
3467  // NOTE: vs2022-32bit-release builds were giving wrong
3468  // results in this block, if it was written as either
3469  // as a switch(num_newl) or its equivalent if-form.
3470  //
3471  // For this reason, we're using a dedicated function
3472  // (**_compress), which seems to work around the issue.
3473  //
3474  // The manifested problem was that somewhere between the
3475  // assignment to curr and this point, proc.wpos (the
3476  // write-position of the processor) jumped to npos, which
3477  // made the write wrap-around! To make things worse,
3478  // enabling prints via _c4dbgpf() and _c4dbgfbf() made the
3479  // problem go away!
3480  //
3481  // The only way to make the problem appear with prints
3482  // enabled was by disabling all prints in this function
3483  // (including in the block which was moved to the compress
3484  // function) and then selectively enabling only some of
3485  // those prints.
3486  //
3487  // This may be due to some bug in the cl-x86 optimizer; or
3488  // it may be triggered by some UB which may be
3489  // inadvertedly present in this function or in the filter
3490  // processor. This is despite our best efforts to weed out
3491  // any such UB problem: neither clang-tidy nor none of the
3492  // sanitizers, or gcc's -fanalyzer pointed to any problems
3493  // in this code.
3494  //
3495  // In the end, moving this block to a separate function
3496  // was the only way to bury the problem. But it may
3497  // resurface again, as The Undead, rising to from the
3498  // grave to haunt us with his terrible presence.
3499  //
3500  // We may have to revisit this. With a stake, and lots of
3501  // garlic.
3502  wpos_at_first_newl = _filter_block_folded_newlines_compress(proc, ++num_newl, wpos_at_first_newl);
3503  _filter_block_indentation(proc, indentation);
3504  break;
3505  }
3506  case ' ':
3507  case '\t':
3508  {
3509  size_t first = proc.rem().first_not_of(" \t");
3510  _c4dbgfbf("space. first={}", first);
3511  if(first == npos)
3512  first = proc.rem().len;
3513  _c4dbgfbf("... indentation increased to {}", first);
3514  if(num_newl)
3515  {
3516  _c4dbgfbf("... prev space (at wpos={}) must be newline", wpos_at_first_newl);
3517  proc.set_at(wpos_at_first_newl, '\n');
3518  }
3519  if(num_newl > 1u)
3520  {
3521  _c4dbgfbf("... add missing newline", wpos_at_first_newl);
3522  proc.set('\n');
3523  }
3524  _filter_block_folded_indented_block(proc, indentation, len, first);
3525  num_newl = 0;
3526  wpos_at_first_newl = npos;
3527  break;
3528  }
3529  case '\r':
3530  proc.skip();
3531  break;
3532  default:
3533  _c4dbgfbf("not space, not newline. stop.", 0);
3534  return;
3535  }
3536  }
3537 }
3538 
3539 
3540 template<class EventHandler>
3541 template<class FilterProcessor>
3542 void ParseEngine<EventHandler>::_filter_block_folded_indented_block(FilterProcessor &C4_RESTRICT proc, size_t indentation, size_t len, size_t curr_indentation) noexcept
3543 {
3544  RYML_ASSERT_PARSE_CB_(this->callbacks(), (proc.rem().first_not_of(" \t") == curr_indentation) || (proc.rem().first_not_of(" \t") == npos), m_evt_handler->m_curr->pos);
3545  if(curr_indentation)
3546  proc.copy(curr_indentation);
3547  while(proc.has_more_chars(len))
3548  {
3549  const char curr = proc.curr();
3550  _c4dbgfbf("'{}' sofar={}", _c4prc(curr), prs_(proc.sofar()));
3551  switch(curr)
3552  {
3553  case '\n':
3554  {
3555  proc.copy();
3556  _filter_block_indentation(proc, indentation);
3557  csubstr rem = proc.rem();
3558  const size_t first = rem.first_not_of(' ');
3559  _c4dbgfbf("newline. firstns={}", first);
3560  if(first == 0)
3561  {
3562  const char c = rem[first];
3563  _c4dbgfbf("firstns={}='{}'", first, _c4prc(c));
3564  if(c != '\n' && c != '\r')
3565  {
3566  _c4dbgfbf("done with indented block", first);
3567  goto endloop;
3568  }
3569  }
3570  else if(first != npos)
3571  {
3572  proc.copy(first);
3573  _c4dbgfbf("copy all {} spaces", first);
3574  }
3575  break;
3576  }
3577  break;
3578  case '\r':
3579  proc.skip();
3580  break;
3581  default:
3582  proc.copy();
3583  break;
3584  }
3585  }
3586  endloop:
3587  return;
3588 }
3589 
3590 
3591 template<class EventHandler>
3592 template<class FilterProcessor>
3593 auto ParseEngine<EventHandler>::_filter_block_folded(FilterProcessor &C4_RESTRICT proc, size_t indentation, BlockChomp_e chomp) -> decltype(proc.result())
3594 {
3595  _c4dbgfbf("indentation={} before={}", indentation, prs_(proc.src));
3596 
3597  size_t contents_len = _handle_all_whitespace(proc, chomp);
3598  if(!contents_len)
3599  return proc.result();
3600 
3601  contents_len = _extend_to_chomp(proc, contents_len);
3602 
3603  _c4dbgfbf("to filter={}", prs_(proc.src.first(contents_len)));
3604 
3605  _filter_block_folded_newlines_leading(proc, indentation, contents_len);
3606 
3607  // now filter the bulk
3608  while(proc.has_more_chars(/*maxpos*/contents_len))
3609  {
3610  const char curr = proc.curr();
3611  _c4dbgfbf("'{}' sofar={}", _c4prc(curr), prs_(proc.sofar()));
3612  switch(curr)
3613  {
3614  case '\n':
3615  {
3616  _c4dbgfbf("found newline", curr);
3617  _filter_block_folded_newlines(proc, indentation, contents_len);
3618  break;
3619  }
3620  case '\r':
3621  proc.skip();
3622  break;
3623  default:
3624  proc.copy();
3625  break;
3626  }
3627  }
3628 
3629  _c4dbgfbf("before chomp: #tochomp={} sofar={}", proc.rem().len, prs_(proc.sofar()));
3630 
3631  _filter_chomp(proc, chomp, indentation);
3632 
3633  _c4dbgfbf("final={}", proc.sofar().len, prs_(proc.sofar()));
3634 
3635  return proc.result();
3636 }
3637 
3638 #undef _c4dbgfbf
3639 
3640 template<class EventHandler>
3641 FilterResult ParseEngine<EventHandler>::filter_scalar_block_folded(csubstr scalar, substr dst, size_t indentation, BlockChomp_e chomp)
3642 {
3643  FilterProcessorSrcDst proc(scalar, dst);
3644  return _filter_block_folded(proc, indentation, chomp);
3645 }
3646 
3647 template<class EventHandler>
3648 FilterResult ParseEngine<EventHandler>::filter_scalar_block_folded_in_place(substr scalar, size_t cap, size_t indentation, BlockChomp_e chomp)
3649 {
3650  FilterProcessorInplaceEndExtending proc(scalar, cap);
3651  return _filter_block_folded(proc, indentation, chomp);
3652 }
3653 
3654 
3655 //-----------------------------------------------------------------------------
3656 //-----------------------------------------------------------------------------
3657 //-----------------------------------------------------------------------------
3658 
3659 template<class EventHandler>
3660 csubstr ParseEngine<EventHandler>::_filter_scalar_plain(substr s, size_t indentation)
3661 {
3662  _c4dbgpf("filtering plain scalar: s={}", prs_(s));
3663  FilterResult r = this->filter_scalar_plain_in_place(s, s.len, indentation);
3664  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, r.valid(), m_evt_handler->m_curr->pos);
3665  _c4dbgpf("filtering plain scalar: success! s={}", prs_(r.get()));
3666  return r.get();
3667 }
3668 
3669 //-----------------------------------------------------------------------------
3670 
3671 template<class EventHandler>
3672 csubstr ParseEngine<EventHandler>::_filter_scalar_squot(substr s)
3673 {
3674  _c4dbgpf("filtering squo scalar: s={}", prs_(s));
3675  FilterResult r = this->filter_scalar_squoted_in_place(s, s.len);
3676  RYML_ASSERT_PARSE_CB_(this->callbacks(), r.valid(), m_evt_handler->m_curr->pos);
3677  _c4dbgpf("filtering squo scalar: success! s={}", prs_(r.get()));
3678  return r.get();
3679 }
3680 
3681 
3682 //-----------------------------------------------------------------------------
3683 
3684 template<class EventHandler>
3685 csubstr ParseEngine<EventHandler>::_filter_scalar_dquot(substr s)
3686 {
3687  _c4dbgpf("filtering dquo scalar: s={}", prs_(s));
3688  FilterResultExtending r = this->filter_scalar_dquoted_in_place(s, s.len);
3689  if C4_LIKELY(r.valid())
3690  {
3691  _c4dbgpf("filtering dquo scalar: success! s={}", prs_(r.get()));
3692  return r.get();
3693  }
3694  else
3695  {
3696  const size_t len = r.required_len();
3697  _c4dbgpf("filtering dquo scalar: not enough space: needs {}, have {}", len, s.len);
3698  substr dst = _alloc_arena(len, &s);
3699  _c4dbgpf("filtering dquo scalar: dst.len={}", dst.len);
3700  if(dst.str)
3701  {
3702  RYML_ASSERT_PARSE_CB_(this->callbacks(), dst.len == len, m_evt_handler->m_curr->pos);
3703  FilterResult rsd = this->filter_scalar_dquoted(s, dst);
3704  _c4dbgpf("filtering dquo scalar: ... result now needs {} was {}", rsd.required_len(), len);
3705  RYML_ASSERT_PARSE_CB_(this->callbacks(), rsd.required_len() <= len, m_evt_handler->m_curr->pos); // may be smaller!
3706  RYML_CHECK_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, rsd.valid(), m_evt_handler->m_curr->pos);
3707  _c4dbgpf("filtering dquo scalar: success! s={}", prs_(rsd.get()));
3708  return rsd.get();
3709  }
3710  return dst;
3711  }
3712 }
3713 
3714 
3715 //-----------------------------------------------------------------------------
3716 
3717 template<class EventHandler>
3718 csubstr ParseEngine<EventHandler>::_move_scalar_left_and_add_newline(substr s)
3719 {
3720  if(s.is_sub(_buf()))
3721  {
3722  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.str > _buf().str, m_evt_handler->m_curr->pos);
3723  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.str-1 >= _buf().str, m_evt_handler->m_curr->pos);
3724  if(s.len)
3725  memmove(s.str - 1, s.str, s.len);
3726  --s.str;
3727  s.str[s.len] = '\n';
3728  ++s.len;
3729  return s;
3730  }
3731  else
3732  {
3733  substr dst = _alloc_arena(s.len + 1, &s);
3734  if(s.len)
3735  memcpy(dst.str, s.str, s.len);
3736  dst[s.len] = '\n';
3737  return dst;
3738  }
3739 }
3740 
3741 template<class EventHandler>
3742 csubstr ParseEngine<EventHandler>::_filter_scalar_literal(substr s, size_t indentation, BlockChomp_e chomp)
3743 {
3744  _c4dbgpf("filtering block literal scalar: s={}", prs_(s));
3745  FilterResult r = this->filter_scalar_block_literal_in_place(s, s.len, indentation, chomp);
3746  csubstr result;
3747  if C4_LIKELY(r.valid())
3748  {
3749  result = r.get();
3750  }
3751  else
3752  {
3753  _c4dbgpf("filtering block literal scalar: not enough space: needs {}, have {}", r.required_len(), s.len);
3754  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, r.required_len() == s.len + 1, m_evt_handler->m_curr->pos);
3755  // this can only happen when adding a single newline in clip mode.
3756  // so we shift left the scalar by one place
3757  result = _move_scalar_left_and_add_newline(s);
3758  }
3759  _c4dbgpf("filtering block literal scalar: success! s={}", prs_(result));
3760  return result;
3761 }
3762 
3763 
3764 //-----------------------------------------------------------------------------
3765 template<class EventHandler>
3766 csubstr ParseEngine<EventHandler>::_filter_scalar_folded(substr s, size_t indentation, BlockChomp_e chomp)
3767 {
3768  _c4dbgpf("filtering block folded scalar: s={}", prs_(s));
3769  FilterResult r = this->filter_scalar_block_folded_in_place(s, s.len, indentation, chomp);
3770  csubstr result;
3771  if C4_LIKELY(r.valid())
3772  {
3773  result = r.get();
3774  }
3775  else
3776  {
3777  _c4dbgpf("filtering block folded scalar: not enough space: needs {}, have {}", r.required_len(), s.len);
3778  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, r.required_len() == s.len + 1, m_evt_handler->m_curr->pos);
3779  // this can only happen when adding a single newline in clip mode.
3780  // so we shift left the scalar by one place
3781  result = _move_scalar_left_and_add_newline(s);
3782  }
3783  _c4dbgpf("filtering block folded scalar: success! s={}", prs_(result));
3784  return result;
3785 }
3786 
3787 
3788 //-----------------------------------------------------------------------------
3789 
3790 template<class EventHandler>
3791 csubstr ParseEngine<EventHandler>::_maybe_filter_key_scalar_plain(ScannedScalar const& C4_RESTRICT sc, size_t indentation)
3792 {
3793  if(sc.needs_filter)
3794  {
3795  if(m_options.scalar_filtering())
3796  {
3797  return _filter_scalar_plain(sc.scalar, indentation);
3798  }
3799  else
3800  {
3801  _c4dbgp("plain scalar left unfiltered");
3802  m_evt_handler->mark_key_scalar_unfiltered();
3803  }
3804  }
3805  else
3806  {
3807  _c4dbgp("plain scalar doesn't need filtering");
3808  }
3809  return sc.scalar;
3810 }
3811 
3812 template<class EventHandler>
3813 csubstr ParseEngine<EventHandler>::_maybe_filter_val_scalar_plain(ScannedScalar const& C4_RESTRICT sc, size_t indentation)
3814 {
3815  if(sc.needs_filter)
3816  {
3817  if(m_options.scalar_filtering())
3818  {
3819  return _filter_scalar_plain(sc.scalar, indentation);
3820  }
3821  else
3822  {
3823  _c4dbgp("plain scalar left unfiltered");
3824  m_evt_handler->mark_val_scalar_unfiltered();
3825  }
3826  }
3827  else
3828  {
3829  _c4dbgp("plain scalar doesn't need filtering");
3830  }
3831  return sc.scalar;
3832 }
3833 
3834 
3835 //-----------------------------------------------------------------------------
3836 
3837 template<class EventHandler>
3838 csubstr ParseEngine<EventHandler>::_maybe_filter_key_scalar_squot(ScannedScalar const& C4_RESTRICT sc)
3839 {
3840  if(sc.needs_filter)
3841  {
3842  if(m_options.scalar_filtering())
3843  {
3844  return _filter_scalar_squot(sc.scalar);
3845  }
3846  else
3847  {
3848  _c4dbgp("squo key scalar left unfiltered");
3849  m_evt_handler->mark_key_scalar_unfiltered();
3850  }
3851  }
3852  else
3853  {
3854  _c4dbgp("squo key scalar doesn't need filtering");
3855  }
3856  return sc.scalar;
3857 }
3858 
3859 template<class EventHandler>
3860 csubstr ParseEngine<EventHandler>::_maybe_filter_val_scalar_squot(ScannedScalar const& C4_RESTRICT sc)
3861 {
3862  if(sc.needs_filter)
3863  {
3864  if(m_options.scalar_filtering())
3865  {
3866  return _filter_scalar_squot(sc.scalar);
3867  }
3868  else
3869  {
3870  _c4dbgp("squo val scalar left unfiltered");
3871  m_evt_handler->mark_val_scalar_unfiltered();
3872  }
3873  }
3874  else
3875  {
3876  _c4dbgp("squo val scalar doesn't need filtering");
3877  }
3878  return sc.scalar;
3879 }
3880 
3881 
3882 //-----------------------------------------------------------------------------
3883 
3884 template<class EventHandler>
3885 csubstr ParseEngine<EventHandler>::_maybe_filter_key_scalar_dquot(ScannedScalar const& C4_RESTRICT sc)
3886 {
3887  if(sc.needs_filter)
3888  {
3889  if(m_options.scalar_filtering())
3890  {
3891  return _filter_scalar_dquot(sc.scalar);
3892  }
3893  else
3894  {
3895  _c4dbgp("dquo scalar left unfiltered");
3896  m_evt_handler->mark_key_scalar_unfiltered();
3897  }
3898  }
3899  else
3900  {
3901  _c4dbgp("dquo scalar doesn't need filtering");
3902  }
3903  return sc.scalar;
3904 }
3905 
3906 template<class EventHandler>
3907 csubstr ParseEngine<EventHandler>::_maybe_filter_val_scalar_dquot(ScannedScalar const& C4_RESTRICT sc)
3908 {
3909  if(sc.needs_filter)
3910  {
3911  if(m_options.scalar_filtering())
3912  {
3913  return _filter_scalar_dquot(sc.scalar);
3914  }
3915  else
3916  {
3917  _c4dbgp("dquo scalar left unfiltered");
3918  m_evt_handler->mark_val_scalar_unfiltered();
3919  }
3920  }
3921  else
3922  {
3923  _c4dbgp("dquo scalar doesn't need filtering");
3924  }
3925  return sc.scalar;
3926 }
3927 
3928 
3929 //-----------------------------------------------------------------------------
3930 
3931 template<class EventHandler>
3932 csubstr ParseEngine<EventHandler>::_maybe_filter_key_scalar_literal(ScannedBlock const& C4_RESTRICT sb)
3933 {
3934  if(m_options.scalar_filtering())
3935  {
3936  return _filter_scalar_literal(sb.scalar, sb.indentation, sb.chomp);
3937  }
3938  else
3939  {
3940  _c4dbgp("literal scalar left unfiltered");
3941  m_evt_handler->mark_key_scalar_unfiltered();
3942  }
3943  return sb.scalar;
3944 }
3945 
3946 template<class EventHandler>
3947 csubstr ParseEngine<EventHandler>::_maybe_filter_val_scalar_literal(ScannedBlock const& C4_RESTRICT sb)
3948 {
3949  if(m_options.scalar_filtering())
3950  {
3951  return _filter_scalar_literal(sb.scalar, sb.indentation, sb.chomp);
3952  }
3953  else
3954  {
3955  _c4dbgp("literal scalar left unfiltered");
3956  m_evt_handler->mark_val_scalar_unfiltered();
3957  }
3958  return sb.scalar;
3959 }
3960 
3961 
3962 //-----------------------------------------------------------------------------
3963 
3964 template<class EventHandler>
3965 csubstr ParseEngine<EventHandler>::_maybe_filter_key_scalar_folded(ScannedBlock const& C4_RESTRICT sb)
3966 {
3967  if(m_options.scalar_filtering())
3968  {
3969  return _filter_scalar_folded(sb.scalar, sb.indentation, sb.chomp);
3970  }
3971  else
3972  {
3973  _c4dbgp("folded scalar left unfiltered");
3974  m_evt_handler->mark_key_scalar_unfiltered();
3975  }
3976  return sb.scalar;
3977 }
3978 
3979 template<class EventHandler>
3980 csubstr ParseEngine<EventHandler>::_maybe_filter_val_scalar_folded(ScannedBlock const& C4_RESTRICT sb)
3981 {
3982  if(m_options.scalar_filtering())
3983  {
3984  return _filter_scalar_folded(sb.scalar, sb.indentation, sb.chomp);
3985  }
3986  else
3987  {
3988  _c4dbgp("folded scalar left unfiltered");
3989  m_evt_handler->mark_val_scalar_unfiltered();
3990  }
3991  return sb.scalar;
3992 }
3993 
3994 
3995 //-----------------------------------------------------------------------------
3996 //-----------------------------------------------------------------------------
3997 //-----------------------------------------------------------------------------
3998 
3999 #ifdef RYML_DBG // !!! <----------------------------------
4000 
4001 template<class EventHandler>
4002 void ParseEngine<EventHandler>::add_flags(ParserFlag_t on)
4003 {
4004  ParserState *s = m_evt_handler->m_curr;
4005  char buf1_[64], buf2_[64], buf3_[64];
4006  csubstr buf1 = detail::_parser_flags_to_str(buf1_, on);
4007  csubstr buf2 = detail::_parser_flags_to_str(buf2_, s->flags);
4008  csubstr buf3 = detail::_parser_flags_to_str(buf3_, s->flags|on);
4009  _c4dbgpf("state[{}]: add {}: before={} after={}", s->level, buf1, buf2, buf3);
4010  s->flags |= on;
4011 }
4012 
4013 template<class EventHandler>
4014 void ParseEngine<EventHandler>::addrem_flags(ParserFlag_t on, ParserFlag_t off)
4015 {
4016  ParserState *s = m_evt_handler->m_curr;
4017  char buf1_[64], buf2_[64], buf3_[64], buf4_[64];
4018  csubstr buf1 = detail::_parser_flags_to_str(buf1_, on);
4019  csubstr buf2 = detail::_parser_flags_to_str(buf2_, off);
4020  csubstr buf3 = detail::_parser_flags_to_str(buf3_, s->flags);
4021  csubstr buf4 = detail::_parser_flags_to_str(buf4_, (~off)&((s->flags|on)));
4022  _c4dbgpf("state[{}]: add {} / rem {}: before={} after={}", s->level, buf1, buf2, buf3, buf4);
4023  RYML_ASSERT_BASIC_((on & off) == ParserFlag_t(0));
4024  s->flags &= ~off;
4025  s->flags |= on;
4026 }
4027 
4028 template<class EventHandler>
4029 void ParseEngine<EventHandler>::rem_flags(ParserFlag_t off)
4030 {
4031  ParserState *s = m_evt_handler->m_curr;
4032  char buf1_[64], buf2_[64], buf3_[64];
4033  csubstr buf1 = detail::_parser_flags_to_str(buf1_, off);
4034  csubstr buf2 = detail::_parser_flags_to_str(buf2_, s->flags);
4035  csubstr buf3 = detail::_parser_flags_to_str(buf3_, s->flags&(~off));
4036  _c4dbgpf("state[{}]: rem {}: before={} after={}", s->level, buf1, buf2, buf3);
4037  s->flags &= ~off;
4038 }
4039 
4040 inline C4_NO_INLINE csubstr detail::_parser_flags_to_str(substr buf, ParserFlag_t flags)
4041 {
4042  size_t pos = 0;
4043  bool gotone = false;
4044 
4045  #define _prflag(fl) \
4046  if((flags & fl) == (fl)) \
4047  { \
4048  if(gotone) \
4049  { \
4050  if(pos + 1 < buf.len) \
4051  buf[pos] = '|'; \
4052  ++pos; \
4053  } \
4054  csubstr fltxt = #fl; \
4055  if(pos + fltxt.len <= buf.len) \
4056  memcpy(buf.str + pos, fltxt.str, fltxt.len); \
4057  pos += fltxt.len; \
4058  gotone = true; \
4059  }
4060 
4061  _prflag(RTOP);
4062  _prflag(RUNK);
4063  _prflag(RMAP);
4064  _prflag(RSEQ);
4065  _prflag(RFLOW);
4066  _prflag(RBLCK);
4067  _prflag(QMRK);
4068  _prflag(RKEY);
4069  _prflag(RVAL);
4070  _prflag(RKCL);
4071  _prflag(RNXT);
4072  _prflag(SSCL);
4073  _prflag(QSCL);
4074  _prflag(RSET);
4075  _prflag(RDOC);
4076  _prflag(NDOC);
4077  _prflag(USTY);
4078  _prflag(RSEQIMAP);
4079 
4080  #undef _prflag
4081 
4082  if(pos == 0)
4083  if(buf.len > 0)
4084  buf[pos++] = '0';
4085 
4086  RYML_CHECK_BASIC_(pos <= buf.len);
4087 
4088  return buf.first(pos);
4089 }
4090 
4091 #endif // RYML_DBG !!! <----------------------------------
4092 
4093 
4094 //-----------------------------------------------------------------------------
4095 //-----------------------------------------------------------------------------
4096 //-----------------------------------------------------------------------------
4097 
4098 template<class EventHandler>
4099 csubstr ParseEngine<EventHandler>::location_contents(Location const& loc) const
4100 {
4101  RYML_ASSERT_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, loc.offset < _buf().len);
4102  return _buf().sub(loc.offset);
4103 }
4104 
4105 template<class EventHandler>
4106 Location ParseEngine<EventHandler>::val_location(const char *val) const
4107 {
4108  if C4_UNLIKELY(val == nullptr)
4109  return {m_evt_handler->m_curr->pos.name, 0, 0, 0};
4110  RYML_CHECK_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, m_options.locations());
4111  // NOTE: if any of these checks fails, the parser needs to be
4112  // instantiated with locations enabled.
4113  RYML_ASSERT_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, m_options.locations());
4114  RYML_ASSERT_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, !_locations_dirty());
4115  RYML_ASSERT_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, m_newline_offsets != nullptr);
4116  RYML_ASSERT_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, m_newline_offsets_size > 0);
4117  // NOTE: the pointer needs to belong to the buffer that was used to parse.
4118  csubstr src = _buf();
4119  RYML_CHECK_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, val != nullptr || src.str == nullptr);
4120  RYML_CHECK_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, (val >= src.begin() && val <= src.end()) || (src.str == nullptr && val == nullptr));
4121  // ok. search the first stored newline after the given ptr
4122  using lineptr_type = size_t const* C4_RESTRICT;
4123  lineptr_type lineptr = nullptr;
4124  size_t offset = (size_t)(val - src.begin());
4125  if(m_newline_offsets_size < RYML_LOCATIONS_SMALL_THRESHOLD)
4126  {
4127  // just do a linear search if the size is small.
4128  for(lineptr_type curr = m_newline_offsets, last = m_newline_offsets + m_newline_offsets_size; curr < last; ++curr)
4129  {
4130  if(*curr > offset)
4131  {
4132  lineptr = curr;
4133  break;
4134  }
4135  }
4136  }
4137  else
4138  {
4139  // do a bisection search if the size is not small.
4140  //
4141  // We could use std::lower_bound but this is simple enough and
4142  // spares the costly include of <algorithm>.
4143  size_t count = m_newline_offsets_size;
4144  lineptr = m_newline_offsets;
4145  while(count)
4146  {
4147  size_t step = count >> 1;
4148  lineptr_type it = lineptr + step;
4149  if(*it < offset)
4150  {
4151  lineptr = ++it;
4152  count -= step + 1;
4153  }
4154  else
4155  {
4156  count = step;
4157  }
4158  }
4159  }
4160  RYML_ASSERT_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, lineptr);
4161  RYML_ASSERT_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, lineptr >= m_newline_offsets);
4162  RYML_ASSERT_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, lineptr <= m_newline_offsets + m_newline_offsets_size);
4163  RYML_ASSERT_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, lineptr && (*lineptr > offset));
4164  Location loc;
4165  loc.name = m_evt_handler->m_curr->pos.name;
4166  loc.offset = offset;
4167  loc.line = (size_t)(lineptr - m_newline_offsets);
4168  if(lineptr > m_newline_offsets)
4169  loc.col = (offset - *(lineptr-1) - 1u);
4170  else
4171  loc.col = offset;
4172  return loc;
4173 }
4174 
4175 template<class EventHandler>
4176 void ParseEngine<EventHandler>::_prepare_locations()
4177 {
4178  csubstr src = _buf();
4179  size_t numnewlines = 1u + src.count('\n');
4180  _resize_locations(numnewlines);
4181  m_newline_offsets_size = 0;
4182  for(size_t i = 0; i < src.len; i++)
4183  if(src.str[i] == '\n')
4184  m_newline_offsets[m_newline_offsets_size++] = i; // NOLINT
4185  m_newline_offsets[m_newline_offsets_size++] = src.len; // NOLINT
4186  RYML_ASSERT_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, m_newline_offsets_size == numnewlines);
4187 }
4188 
4189 template<class EventHandler>
4190 void ParseEngine<EventHandler>::_resize_locations(size_t numnewlines)
4191 {
4192  numnewlines = numnewlines >= 16 ? numnewlines : 16;
4193  if(numnewlines > m_newline_offsets_capacity)
4194  {
4195  if(m_newline_offsets)
4196  RYML_CB_FREE_(m_evt_handler->m_stack.m_callbacks, m_newline_offsets, size_t, m_newline_offsets_capacity);
4197  m_newline_offsets = RYML_CB_ALLOC_HINT_(m_evt_handler->m_stack.m_callbacks, size_t, numnewlines, m_newline_offsets);
4198  m_newline_offsets_capacity = numnewlines;
4199  }
4200 }
4201 
4202 template<class EventHandler>
4203 bool ParseEngine<EventHandler>::_locations_dirty() const
4204 {
4205  return !m_newline_offsets_size;
4206 }
4207 
4208 
4209 //-----------------------------------------------------------------------------
4210 //-----------------------------------------------------------------------------
4211 //-----------------------------------------------------------------------------
4212 
4213 template<class EventHandler>
4214 void ParseEngine<EventHandler>::_handle_flow_skip_whitespace()
4215 {
4216  // don't assign to csubstr rem: otherwise, gcc12,13,14 -O3 -m32 misbuilds
4217  if(m_evt_handler->m_curr->line_contents.rem.len > 0)
4218  {
4219  if(m_evt_handler->m_curr->line_contents.rem.str[0] == ' ' || m_evt_handler->m_curr->line_contents.rem.str[0] == '\t')
4220  {
4221  _c4dbgpf("starts with whitespace: '{}'", _c4prc(m_evt_handler->m_curr->line_contents.rem.str[0]));
4222  _skipchars(" \t");
4223  }
4224  // comments
4225  if(m_evt_handler->m_curr->line_contents.rem.begins_with('#'))
4226  {
4227  _c4dbgpf("it's a comment: {}", m_evt_handler->m_curr->line_contents.rem);
4228  _line_progressed(m_evt_handler->m_curr->line_contents.rem.len);
4229  }
4230  }
4231 }
4232 
4233 
4234 template<class EventHandler>
4235 void ParseEngine<EventHandler>::_handle_flow_line_beginning()
4236 {
4237  _c4dbgpf("flow: indref={} indentation={}", m_evt_handler->m_curr->indref, m_evt_handler->m_curr->line_contents.indentation);
4238  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->at_line_beginning(), m_evt_handler->m_curr->pos);
4239  if C4_UNLIKELY(m_evt_handler->m_curr->indentation_lt())
4240  {
4241  csubstr trimmed = m_evt_handler->m_curr->line_contents.rem.sub(m_evt_handler->m_curr->line_contents.indentation);
4242  _c4dbgpf("flow: after indentation={}", prs_(trimmed));
4243  if(trimmed.len && trimmed.triml(" \t").len)
4244  {
4245  _line_progressed(m_evt_handler->m_curr->line_contents.indentation);
4246  _c4err("bad indentation");
4247  }
4248  }
4249 }
4250 
4251 template<class EventHandler>
4252 size_t ParseEngine<EventHandler>::_handle_block_skip_leading_whitespace()
4253 {
4254  const size_t mark = m_evt_handler->m_curr->pos.offset;
4255  const size_t firstpos = m_evt_handler->m_curr->line_contents.rem.first_not_of(" \t");
4256  _c4dbgpf("block: mark={} firstpos={}", mark, firstpos);
4257  if(firstpos != npos)
4258  {
4259  _c4dbgp("block: non empty line");
4260  _line_progressed(firstpos);
4261  return mark;
4262  }
4263  else
4264  {
4265  _c4dbgp("block: rest of line is whitespace");
4266  _line_progressed(m_evt_handler->m_curr->line_contents.rem.len);
4267  return npos;
4268  }
4269 }
4270 
4271 template<class EventHandler>
4272 void ParseEngine<EventHandler>::_handle_block_check_leading_tabs(size_t start_mark, size_t end_mark)
4273 {
4274  _c4dbgpf("block: start_mark={} end_mark={}", start_mark, end_mark);
4275  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, end_mark >= start_mark, m_evt_handler->m_curr->pos);
4276  if(end_mark != start_mark)
4277  {
4278  csubstr leading = _buf().range(start_mark, end_mark);
4279  _c4dbgpf("block: leading[{}-{}]={}", start_mark, end_mark, prs_(leading, true));
4280  size_t pos = leading.find('\t');
4281  if(pos != npos)
4282  {
4283  size_t fno = leading.first_not_of(" \t");
4284  if(fno == npos || pos < fno)
4285  _c4err("invalid tab character to the left");
4286  }
4287  (void)leading;
4288  }
4289 }
4290 
4291 
4292 //-----------------------------------------------------------------------------
4293 
4294 
4295 template<class EventHandler>
4296 void ParseEngine<EventHandler>::_handle_colon()
4297 {
4298  size_t curr = m_evt_handler->m_curr->pos.line;
4299  if C4_UNLIKELY(m_prev_colon != npos && curr == m_prev_colon)
4300  {
4301  _c4dbgpf("colon: prevline={} currline={}", m_prev_colon, curr);
4302  _c4err("two colons on same line");
4303  }
4304  _c4dbgpf("colon: set prevline={}->{}", m_prev_colon, curr);
4305  m_prev_colon = curr;
4306 }
4307 
4308 template<class EventHandler>
4309 void ParseEngine<EventHandler>::_add_annotation(Annotation *C4_RESTRICT dst, csubstr str)
4310 {
4311  _c4dbgpf("store annotation[{}]: {}", dst->num_entries, prs_(str));
4312  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, dst->num_entries < C4_COUNTOF(dst->annotations), m_evt_handler->m_curr->pos); // NOLINT(bugprone-sizeof-expression)
4313  dst->annotations[dst->num_entries].str = str;
4314  dst->annotations[dst->num_entries].indentation = {};
4315  dst->annotations[dst->num_entries].line = {};
4316  dst->annotations[dst->num_entries].orig = {};
4317  ++dst->num_entries;
4318 }
4319 
4320 template<class EventHandler>
4321 void ParseEngine<EventHandler>::_add_annotation(Annotation *C4_RESTRICT dst, csubstr str, size_t indentation, size_t line)
4322 {
4323  _c4dbgpf("store annotation[{}]: '{}' indentation={} line={}", dst->num_entries, maybe_null_str_(str), indentation, line);
4324  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, dst->num_entries < C4_COUNTOF(dst->annotations), m_evt_handler->m_curr->pos); // NOLINT(bugprone-sizeof-expression)
4325  if C4_UNLIKELY(dst->num_entries && dst->annotations[0].line == line)
4326  {
4327  _c4err("parse error");
4328  }
4329  dst->annotations[dst->num_entries].str = str;
4330  dst->annotations[dst->num_entries].indentation = indentation;
4331  dst->annotations[dst->num_entries].line = line;
4332  dst->annotations[dst->num_entries].orig = {};
4333  ++dst->num_entries;
4334 }
4335 
4336 template<class EventHandler>
4337 void ParseEngine<EventHandler>::_add_annotation(Annotation *C4_RESTRICT dst, csubstr str, size_t indentation, size_t line, csubstr orig)
4338 {
4339  _c4dbgpf("store annotation[{}]: '{}'->'{}' indentation={} line={}", dst->num_entries, orig, maybe_null_str_(str), indentation, line);
4340  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, dst->num_entries < C4_COUNTOF(dst->annotations), m_evt_handler->m_curr->pos); // NOLINT(bugprone-sizeof-expression)
4341  if C4_UNLIKELY(dst->num_entries && dst->annotations[0].line == line)
4342  {
4343  _c4err("parse error");
4344  }
4345  dst->annotations[dst->num_entries].str = str;
4346  dst->annotations[dst->num_entries].indentation = indentation;
4347  dst->annotations[dst->num_entries].line = line;
4348  dst->annotations[dst->num_entries].orig = orig;
4349  ++dst->num_entries;
4350 }
4351 
4352 template<class EventHandler>
4353 bool ParseEngine<EventHandler>::_annotations_require_key_container() const
4354 {
4355  return m_pending_tags.num_entries > 1 || m_pending_anchors.num_entries > 1;
4356 }
4357 
4358 template<class EventHandler>
4359 bool ParseEngine<EventHandler>::_handle_annotations_before_unexpected_flow_token_rkey()
4360 {
4361  if(!(m_pending_tags.num_entries | m_pending_anchors.num_entries))
4362  return false;
4363  _c4dbgpf("handle_annotations_before_unexpected_flow_comma_rkey, node={}", m_evt_handler->m_curr->node_id);
4364  if(m_pending_tags.num_entries)
4365  {
4366  _c4dbgpf("handle_annotations_before_unexpected_flow_comma_rkey, #tags={}", m_pending_tags.num_entries);
4367  if C4_LIKELY(m_pending_tags.num_entries == 1)
4368  {
4369  m_evt_handler->set_key_tag(m_pending_tags.annotations[0].str);
4370  _clear_annotations(&m_pending_tags);
4371  }
4372  else
4373  {
4374  _c4err("too many tags");
4375  }
4376  }
4377  if(m_pending_anchors.num_entries)
4378  {
4379  _c4dbgpf("handle_annotations_before_unexpected_flow_comma, #anchors={}", m_pending_tags.num_entries);
4380  if C4_LIKELY(m_pending_anchors.num_entries == 1)
4381  {
4382  m_evt_handler->set_key_anchor(m_pending_anchors.annotations[0].str);
4383  _clear_annotations(&m_pending_anchors);
4384  }
4385  else
4386  {
4387  _c4err("too many anchors");
4388  }
4389  }
4390  m_evt_handler->set_key_scalar_plain_empty();
4391  m_evt_handler->set_val_scalar_plain_empty();
4392  return true;
4393 }
4394 
4395 template<class EventHandler>
4396 void ParseEngine<EventHandler>::_handle_annotations_before_blck_key_scalar()
4397 {
4398  _c4dbgpf("annotations_before_blck_key_scalar, node={}", m_evt_handler->m_curr->node_id);
4399  if(m_pending_tags.num_entries)
4400  {
4401  _c4dbgpf("annotations_before_blck_key_scalar, #tags={}", m_pending_tags.num_entries);
4402  if C4_LIKELY(m_pending_tags.num_entries == 1)
4403  {
4404  m_evt_handler->set_key_tag(m_pending_tags.annotations[0].str);
4405  _clear_annotations(&m_pending_tags);
4406  }
4407  else
4408  {
4409  _c4err("too many tags"); // LCOV_EXCL_LINE
4410  }
4411  }
4412  if(m_pending_anchors.num_entries)
4413  {
4414  _c4dbgpf("annotations_before_blck_key_scalar, #anchors={}", m_pending_anchors.num_entries);
4415  if C4_LIKELY(m_pending_anchors.num_entries == 1)
4416  {
4417  m_evt_handler->set_key_anchor(m_pending_anchors.annotations[0].str);
4418  _clear_annotations(&m_pending_anchors);
4419  }
4420  else
4421  {
4422  _c4err("too many anchors"); // LCOV_EXCL_LINE
4423  }
4424  }
4425 }
4426 
4427 template<class EventHandler>
4428 void ParseEngine<EventHandler>::_handle_annotations_before_blck_val_scalar()
4429 {
4430  _c4dbgpf("annotations_before_blck_val_scalar, node={}", m_evt_handler->m_curr->node_id);
4431  if(m_pending_tags.num_entries)
4432  {
4433  _c4dbgpf("annotations_before_blck_val_scalar, #tags={}", m_pending_tags.num_entries);
4434  if C4_LIKELY(m_pending_tags.num_entries == 1)
4435  {
4436  m_evt_handler->set_val_tag(m_pending_tags.annotations[0].str);
4437  _clear_annotations(&m_pending_tags);
4438  }
4439  else
4440  {
4441  _c4err("too many tags");
4442  }
4443  }
4444  if(m_pending_anchors.num_entries)
4445  {
4446  _c4dbgpf("annotations_before_blck_val_scalar, #anchors={}", m_pending_anchors.num_entries);
4447  if C4_LIKELY(m_pending_anchors.num_entries == 1)
4448  {
4449  m_evt_handler->set_val_anchor(m_pending_anchors.annotations[0].str);
4450  _clear_annotations(&m_pending_anchors);
4451  }
4452  else
4453  {
4454  _c4err("too many anchors");
4455  }
4456  }
4457 }
4458 
4459 template<class EventHandler>
4460 void ParseEngine<EventHandler>::_handle_annotations_before_start_mapblck(size_t current_line)
4461 {
4462  _c4dbgpf("annotations_before_start_mapblck, current_line={}", current_line);
4463  if(m_pending_tags.num_entries == 2)
4464  {
4465  _c4dbgp("2 tags, setting entry 0");
4466  m_evt_handler->set_val_tag(m_pending_tags.annotations[0].str);
4467  }
4468  else if(m_pending_tags.num_entries == 1)
4469  {
4470  _c4dbgpf("1 tag. line={}, curr={}", m_pending_tags.annotations[0].line, current_line);
4471  if(m_pending_tags.annotations[0].line < current_line)
4472  {
4473  _c4dbgp("...tag is for the map. setting it.");
4474  m_evt_handler->set_val_tag(m_pending_tags.annotations[0].str);
4475  _clear_annotations(&m_pending_tags);
4476  }
4477  }
4478  //
4479  if(m_pending_anchors.num_entries == 2)
4480  {
4481  _c4dbgp("2 anchors, setting entry 0");
4482  m_evt_handler->set_val_anchor(m_pending_anchors.annotations[0].str);
4483  }
4484  else if(m_pending_anchors.num_entries == 1)
4485  {
4486  _c4dbgpf("1 anchor. line={}, curr={}", m_pending_anchors.annotations[0].line, current_line);
4487  if(m_pending_anchors.annotations[0].line < current_line)
4488  {
4489  _c4dbgp("...anchor is for the map. setting it.");
4490  m_evt_handler->set_val_anchor(m_pending_anchors.annotations[0].str);
4491  _clear_annotations(&m_pending_anchors);
4492  }
4493  }
4494 }
4495 
4496 template<class EventHandler>
4497 void ParseEngine<EventHandler>::_handle_annotations_before_start_mapblck_as_key()
4498 {
4499  _c4dbgp("annotations_before_start_mapblck_as_key");
4500  switch(m_pending_tags.num_entries)
4501  {
4502  case 1u:
4503  _c4dbgpf("annotations_after_start_mapblck_as_key: 1 tag={} line={} currline=", prs_(m_pending_tags.annotations[0].str), m_pending_tags.annotations[0].line, m_evt_handler->m_curr->pos.line);
4504  if(m_pending_tags.annotations[0].line != m_evt_handler->m_curr->pos.line)
4505  {
4506  _c4dbgp("annotations_after_start_mapblck_as_key: is map tag");
4507  m_evt_handler->set_key_tag(m_pending_tags.annotations[0].str);
4508  _clear_annotations(&m_pending_tags);
4509  }
4510  break;
4511  case 2u:
4512  _c4dbgpf("annotations_after_start_mapblck_as_key: 2 tags: {} -> {}", prs_(m_pending_tags.annotations[0].str), prs_(m_pending_tags.annotations[1].str));
4513  m_evt_handler->set_key_tag(m_pending_tags.annotations[0].str);
4514  break;
4515  }
4516  switch(m_pending_anchors.num_entries)
4517  {
4518  case 1u:
4519  _c4dbgpf("annotations_after_start_mapblck_as_key: 1 anchor={} line={} currline=", m_pending_anchors.annotations[0].str, m_pending_anchors.annotations[0].line, m_evt_handler->m_curr->pos.line);
4520  if(m_pending_anchors.annotations[0].line != m_evt_handler->m_curr->pos.line)
4521  {
4522  _c4dbgp("annotations_after_start_mapblck_as_key: is map anchor");
4523  m_evt_handler->set_key_anchor(m_pending_anchors.annotations[0].str);
4524  _clear_annotations(&m_pending_anchors);
4525  }
4526  break;
4527  case 2u:
4528  _c4dbgpf("annotations_after_start_mapblck_as_key: 2 anchors: {} -> {}", m_pending_anchors.annotations[0].str, m_pending_anchors.annotations[1].str);
4529  m_evt_handler->set_key_anchor(m_pending_anchors.annotations[0].str);
4530  break;
4531  }
4532 }
4533 
4534 template<class EventHandler>
4535 void ParseEngine<EventHandler>::_handle_annotations_and_indentation_after_start_mapblck(size_t key_indentation, size_t key_line)
4536 {
4537  _c4dbgp("annotations_after_start_mapblck");
4538  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_pending_tags.num_entries <= 2, m_evt_handler->m_curr->pos);
4539  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_pending_anchors.num_entries <= 2, m_evt_handler->m_curr->pos);
4540  if(m_pending_anchors.num_entries || m_pending_tags.num_entries)
4541  {
4542  key_indentation = _select_indentation_from_annotations(key_indentation, key_line);
4543  switch(m_pending_tags.num_entries)
4544  {
4545  case 1u:
4546  _c4dbgpf("annotations_after_start_mapblck: 1 tag: {}", prs_(m_pending_tags.annotations[0].str));
4547  m_evt_handler->set_key_tag(m_pending_tags.annotations[0].str);
4548  _clear_annotations(&m_pending_tags);
4549  break;
4550  case 2u:
4551  _c4dbgpf("annotations_after_start_mapblck: 2 tags: {} -> {}", prs_(m_pending_tags.annotations[0].str), prs_(m_pending_tags.annotations[1].str));
4552  m_evt_handler->set_key_tag(m_pending_tags.annotations[1].str);
4553  _clear_annotations(&m_pending_tags);
4554  break;
4555  }
4556  switch(m_pending_anchors.num_entries)
4557  {
4558  case 1u:
4559  _c4dbgpf("annotations_after_start_mapblck: 1 anchors: {} -> {}", m_pending_anchors.annotations[0].str);
4560  m_evt_handler->set_key_anchor(m_pending_anchors.annotations[0].str);
4561  _clear_annotations(&m_pending_anchors);
4562  break;
4563  case 2u:
4564  _c4dbgpf("annotations_after_start_mapblck: 2 anchors: {} -> {}", m_pending_anchors.annotations[0].str, m_pending_anchors.annotations[1].str);
4565  m_evt_handler->set_key_anchor(m_pending_anchors.annotations[1].str);
4566  _clear_annotations(&m_pending_anchors);
4567  break;
4568  }
4569  }
4570  _set_indentation(key_indentation);
4571 }
4572 
4573 template<class EventHandler>
4574 size_t ParseEngine<EventHandler>::_select_indentation_from_annotations(size_t val_indentation, size_t val_line)
4575 {
4576  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_pending_tags.num_entries | m_pending_anchors.num_entries, m_evt_handler->m_curr->pos);
4577  // select the left-most annotation on the max line
4578  auto const *C4_RESTRICT curr = m_pending_anchors.num_entries ? &m_pending_anchors.annotations[0] : &m_pending_tags.annotations[0];
4579  for(size_t i = 0; i < m_pending_anchors.num_entries; ++i)
4580  {
4581  auto const& C4_RESTRICT ann = m_pending_anchors.annotations[i];
4582  if(ann.line > curr->line)
4583  curr = &ann;
4584  else if(ann.indentation < curr->indentation)
4585  curr = &ann;
4586  }
4587  for(size_t j = 0; j < m_pending_tags.num_entries; ++j)
4588  {
4589  auto const& C4_RESTRICT ann = m_pending_tags.annotations[j];
4590  if(ann.line > curr->line)
4591  curr = &ann;
4592  else if(ann.indentation < curr->indentation)
4593  curr = &ann;
4594  }
4595  return curr->line < val_line ? val_indentation : curr->indentation;
4596 }
4597 
4598 template<class EventHandler>
4599 void ParseEngine<EventHandler>::_handle_keyref(csubstr alias)
4600 {
4601  if C4_LIKELY(!(m_pending_anchors.num_entries | m_pending_tags.num_entries))
4602  m_evt_handler->set_key_ref(alias);
4603  else
4604  _c4err("aliases cannot have anchors or tags");
4605 }
4606 
4607 template<class EventHandler>
4608 void ParseEngine<EventHandler>::_handle_valref(csubstr alias)
4609 {
4610  if C4_LIKELY(!(m_pending_anchors.num_entries | m_pending_tags.num_entries))
4611  m_evt_handler->set_val_ref(alias);
4612  else
4613  _c4err("aliases cannot have anchors or tags");
4614 }
4615 
4616 template<class EventHandler>
4617 csubstr ParseEngine<EventHandler>::_resolve_tag(csubstr tag)
4618 {
4619  _c4dbgpf("resolving tag: {} curr_doc={}", prs_(tag), m_evt_handler->m_curr_doc);
4620  _c4assert(tag.is_sub(_buf()));
4621  TagCache::LookupResult ret = m_evt_handler->tag_cache().find(tag, m_evt_handler->m_curr_doc);
4622  if(ret)
4623  {
4624  _c4dbgpf("resolving tag: found in cache[{}]: {}", ret.pos, prs_(ret.resolved));
4625  return ret.resolved;
4626  }
4627  _c4dbgpf("resolving tag: not in cache: {} curr_doc={}", prs_(tag), m_evt_handler->m_curr_doc);
4628  size_t bufsz = 0;
4629  substr buf = m_evt_handler->arena_rem();
4630  TagDirectives const& C4_RESTRICT tds = m_evt_handler->tag_directives();
4631  csubstr ttag = tds.resolve(buf, &bufsz, tag, m_evt_handler->m_curr_doc,
4632  m_evt_handler->m_curr->pos,
4633  m_evt_handler->m_stack.m_callbacks);
4634  _c4dbgpf("resolving tag: bufsz={} ttag.len={} !!ttag.str={}", bufsz, ttag.len, !!ttag.str);
4635  _c4assert((bufsz > buf.len) == (!ttag.str));
4636  _c4assert(!!bufsz == (ttag.len == bufsz));
4637  // try again if the arena size was not enough
4638  if(!ttag.str)
4639  {
4640  _c4dbgpf("tag requires arena, but it was small. arena.len={} arena.slack={} tag.required={}", m_evt_handler->arena_rem().len, m_evt_handler->arena().len, ttag.len);
4641  _c4assert(ttag.len == bufsz);
4642  buf = _alloc_arena(bufsz, &tag);
4643  if(buf.str) // the alloc may fail eg with the ints handler
4644  {
4645  ttag = tds.resolve(buf, &bufsz, tag, m_evt_handler->m_curr_doc,
4646  m_evt_handler->m_curr->pos,
4647  m_evt_handler->m_stack.m_callbacks);
4648  }
4649  _c4assert(ttag.len == bufsz);
4650  _c4assert(!ttag.str || ttag.is_sub(m_evt_handler->arena()));
4651  }
4652  else if(bufsz) // if we succeeded writing into the arena, grow it as needed
4653  {
4654  _c4dbgp("tag required arena. update size");
4655  _c4assert(ttag.len == bufsz);
4656  _c4assert(ttag.is_sub(buf));
4657  (void)_alloc_arena(bufsz);
4658  }
4659  C4_SUPPRESS_WARNING_MSVC_WITH_PUSH(4127) // conditional expression is constant
4660  if C4_IF_CONSTEXPR (EventHandler::requires_strings_on_buffers) // NOLINT
4661  {
4662  _c4dbgpf("handler requires tags in buffers. !!ttag.str={} in_arena={} in_src={}", !!ttag.str, ttag.is_sub(m_evt_handler->arena()), ttag.is_sub(_buf()));
4663  // is the resolved tag not in any of those buffers?
4664  if(ttag.str && !ttag.is_sub(m_evt_handler->arena()) && !ttag.is_sub(_buf()))
4665  {
4666  _c4dbgpf("copying resolved tag to arena: slack={} required={}", m_evt_handler->arena_rem().len, ttag.len);
4667  buf = _alloc_arena(ttag.len, &tag);
4668  if(buf.str) // the alloc may fail eg with the ints handler
4669  memcpy(buf.str, ttag.str, ttag.len);
4670  ttag.str = buf.str; // keep the current len!
4671  _c4assert(!ttag.str || ttag.is_sub(m_evt_handler->arena()));
4672  }
4673  }
4674  C4_SUPPRESS_WARNING_MSVC_POP
4675  _c4dbgpf("resolved tag: {} --> [{}]~~~{}~~~", prs_(tag), ttag.len, maybe_null_str_(ttag));
4676  _c4assert(ttag.len > 0);
4677  // cache the hard-earned result!
4678  m_evt_handler->tag_cache().add(tag, ttag, m_evt_handler->m_curr_doc, ret.pos);
4679  return ttag;
4680 }
4681 
4682 template<class EventHandler>
4683 bool ParseEngine<EventHandler>::_validate_directive_yaml(csubstr *C4_RESTRICT directive, csubstr *C4_RESTRICT version) const
4684 {
4685  _c4assert(directive->begins_with("%YAML"));
4686  size_t version_start = directive->first_not_of(" \t", 5);
4687  if(version_start != npos)
4688  {
4689  csubstr digits = "0123456789";
4690  size_t major_end = directive->first_not_of(digits, version_start);
4691  if(major_end != npos && directive->str[major_end] == '.') // single dot
4692  {
4693  size_t minor_end = directive->first_not_of(digits, major_end + 1);
4694  if(minor_end == npos)
4695  minor_end = directive->len;
4696  _set_first_strict(*directive, minor_end);
4697  *version = directive->range(version_start, minor_end);
4698  _c4dbgpf("%YAML: version={} full={}", *version, prs_(*directive, true));
4699  return true;
4700  }
4701  }
4702  return false;
4703 }
4704 
4705 template<class EventHandler>
4706 bool ParseEngine<EventHandler>::_validate_directive_tag(csubstr *C4_RESTRICT directive, csubstr *C4_RESTRICT handle, csubstr *C4_RESTRICT prefix) const
4707 {
4708  _c4assert(directive->begins_with("%TAG"));
4709  csubstr whitespace = " \t";
4710  size_t handle_start = directive->first_not_of(whitespace, 4);
4711  if(handle_start != npos && directive->str[handle_start] == '!')
4712  {
4713  size_t handle_end = directive->first_of(whitespace, handle_start);
4714  if(handle_end != npos)
4715  {
4716  size_t prefix_start = directive->first_not_of(whitespace, handle_end);
4717  if(prefix_start != npos)
4718  {
4719  size_t prefix_end = directive->first_of(whitespace, prefix_start);
4720  if(prefix_end == npos)
4721  prefix_end = directive->len;
4722  _set_first_strict(*directive, prefix_end);
4723  *handle = directive->range(handle_start, handle_end);
4724  *prefix = directive->range(prefix_start, prefix_end);
4725  _c4dbgpf("%TAG: handle={} prefix={} full={}", *handle, *prefix, prs_(*directive, true));
4726  if(is_valid_tag_handle(*handle))
4727  return true;
4728  }
4729  }
4730  }
4731  return false;
4732 }
4733 
4734 template<class EventHandler>
4735 void ParseEngine<EventHandler>::_handle_directive(csubstr directive)
4736 {
4737  _c4dbgpf("handle_directive: rem={}", prs_(directive, true));
4738  _c4assert(m_evt_handler->m_curr->line_contents.rem.begins_with('%'));
4739  _c4assert(directive.str == m_evt_handler->m_curr->line_contents.rem.str);
4740  const char *err = nullptr;
4741  csubstr rem;
4742  size_t pos;
4743  auto isdirective = [](csubstr str, csubstr dir) {
4744  if(str.begins_with(dir))
4745  {
4746  csubstr rest = str.sub(dir.len);
4747  return (!rest.len || rest.str[0] == ' ' || rest.str[0] == '\t');
4748  }
4749  return false;
4750  };
4751  if(isdirective(directive, "%TAG"))
4752  {
4753  csubstr handle;
4754  csubstr prefix;
4755  if C4_UNLIKELY(!_validate_directive_tag(&directive, &handle, &prefix))
4756  {
4757  err = "invalid %TAG directive";
4758  goto directive_error; // NOLINT
4759  }
4760  m_evt_handler->add_directive_tag(handle, prefix);
4761  }
4762  else if(isdirective(directive, "%YAML"))
4763  {
4764  csubstr version;
4765  if C4_UNLIKELY(!_validate_directive_yaml(&directive, &version))
4766  {
4767  err = "invalid %YAML directive";
4768  goto directive_error; // NOLINT
4769  }
4770  if C4_UNLIKELY(m_has_directives_yaml)
4771  {
4772  err = "multiple %YAML directives";
4773  goto directive_error; // NOLINT
4774  }
4775  m_has_directives_yaml = true;
4776  m_evt_handler->add_directive_yaml(version);
4777  }
4778  m_has_directives = true;
4779  rem = m_evt_handler->m_curr->line_contents.rem;
4780  pos = rem.first_not_of(" \t", directive.len);
4781  pos = pos != npos ? pos : rem.len;
4782  _line_progressed(pos);
4783  rem = rem.sub(pos);
4784  _c4dbgpf("handle_directive: rest={}", prs_(rem));
4785  if C4_UNLIKELY(rem.len && !rem.begins_with('#'))
4786  {
4787  err = "invalid tokens after directive";
4788  goto directive_error; // NOLINT
4789  }
4790 directive_error:
4791  if C4_UNLIKELY(err != nullptr)
4792  _c4err(err);
4793 }
4794 
4795 template<class EventHandler>
4796 bool ParseEngine<EventHandler>::_handle_bom()
4797 {
4798  const csubstr rem = m_evt_handler->m_curr->line_contents.rem;
4799  if(rem.len)
4800  {
4801  const csubstr rest = rem.sub(1);
4802  // https://yaml.org/spec/1.2.2/#52-character-encodings
4803  #define _rymlisascii(c) ((c) > '\0' && (c) <= '\x7f') // is the character ASCII?
4804  if(rem.begins_with(csubstr{"\x00\x00\xfe\xff", 4})
4805  // no bom:
4806  || (rem.begins_with(csubstr{"\x00\x00\x00", 3}) && rem.len >= 4u && _rymlisascii(rem.str[3])))
4807  {
4808  _c4err("UTF32BE not supported");
4809  }
4810  else if(rem.begins_with(csubstr{"\xff\xfe\x00\x00", 4})
4811  // no bom:
4812  || (rest.begins_with(csubstr{"\x00\x00\x00", 3}) && rem.len >= 4u && _rymlisascii(rem.str[0])))
4813  {
4814  _c4err("UTF32LE not supported");
4815  }
4816  else if(rem.begins_with("\xfe\xff") || (rem.begins_with('\x00') && rem.len >= 2u && _rymlisascii(rem.str[1])))
4817  {
4818  _c4err("UTF16BE not supported");
4819  }
4820  else if(rem.begins_with("\xff\xfe") || (rest.begins_with('\x00') && rem.len >= 2u && _rymlisascii(rem.str[0])))
4821  {
4822  _c4err("UTF16LE not supported");
4823  }
4824  else if(rem.begins_with("\xef\xbb\xbf"))
4825  {
4826  _c4dbgp("byte order mark: UTF8");
4827  _handle_bom(UTF8);
4828  _line_progressed(3);
4829  m_bom_len = 3;
4830  return true;
4831  }
4832  #undef _rymlisascii
4833  }
4834  return false;
4835 }
4836 
4837 template<class EventHandler>
4838 void ParseEngine<EventHandler>::_handle_bom(Encoding_e enc)
4839 {
4840  if(m_encoding == NOBOM)
4841  {
4842  if(enc == UTF8 || /*beginning of file*/(m_evt_handler->m_curr->line_contents.rem.str == _buf().str))
4843  m_encoding = enc;
4844  else
4845  _c4err("non-UTF8 byte order mark can appear only at the beginning of the file"); // LCOV_EXCL_LINE
4846  }
4847  else if(enc != m_encoding)
4848  {
4849  _c4err("byte order mark can only be set once"); // LCOV_EXCL_LINE
4850  }
4851 }
4852 
4853 
4854 //-----------------------------------------------------------------------------
4855 
4856 template<class EventHandler>
4857 void ParseEngine<EventHandler>::_handle_seq_json()
4858 {
4859 seqjson_start:
4860  _c4dbgpf("handle2_seq_json: node_id={} level={} indentation={}", m_evt_handler->m_curr->node_id, m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref);
4861 
4862  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
4863  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RSEQ), m_evt_handler->m_curr->pos);
4864  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RFLOW), m_evt_handler->m_curr->pos);
4865  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RVAL|RNXT), m_evt_handler->m_curr->pos);
4866  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RVAL) != has_all(RNXT), m_evt_handler->m_curr->pos);
4867 
4868  _handle_flow_skip_whitespace();
4869  csubstr rem = m_evt_handler->m_curr->line_contents.rem;
4870  if(!rem.len)
4871  goto seqjson_again;
4872 
4873  if(has_any(RVAL))
4874  {
4875  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
4876  const char first = rem.str[0];
4877  _c4dbgpf("mapjson[RVAL]: '{}'", first);
4878  switch(first)
4879  {
4880  case '"':
4881  {
4882  _c4dbgp("seqjson[RVAL]: scanning double-quoted scalar");
4883  ScannedScalar sc = _scan_scalar_dquot();
4884  csubstr maybe_filtered = _maybe_filter_val_scalar_dquot(sc);
4885  m_evt_handler->set_val_scalar_dquoted(maybe_filtered);
4886  addrem_flags(RNXT, RVAL);
4887  break;
4888  }
4889  case '[':
4890  {
4891  _c4dbgp("seqjson[RVAL]: start child seqjson");
4892  addrem_flags(RNXT, RVAL);
4893  m_evt_handler->begin_seq_val_flow();
4894  addrem_flags(RVAL, RNXT);
4895  _line_progressed(1);
4896  break;
4897  }
4898  case '{':
4899  {
4900  _c4dbgp("seqjson[RVAL]: start child mapjson");
4901  addrem_flags(RNXT, RVAL);
4902  m_evt_handler->begin_map_val_flow();
4903  addrem_flags(RMAP|RKEY, RSEQ|RVAL|RNXT);
4904  _line_progressed(1);
4905  goto seqjson_finish;
4906  }
4907  case ']': // this happens on a trailing comma like ", ]"
4908  {
4909  _c4dbgp("seqjson[RVAL]: end!");
4910  rem_flags(RSEQ);
4911  _end_seq_flow();
4912  _line_progressed(1);
4913  if(!has_all(RSEQ|RFLOW))
4914  goto seqjson_finish;
4915  break;
4916  }
4917  default:
4918  {
4919  ScannedScalar sc;
4920  if(_scan_scalar_seq_json(&sc))
4921  {
4922  _c4dbgp("seqjson[RVAL]: it's a plain scalar.");
4923  csubstr maybe_filtered = _maybe_filter_val_scalar_plain(sc, m_evt_handler->m_curr->indref);
4924  m_evt_handler->set_val_scalar_plain(maybe_filtered);
4925  addrem_flags(RNXT, RVAL);
4926  }
4927  else
4928  {
4929  _c4err("parse error");
4930  }
4931  }
4932  }
4933  }
4934  else // RNXT
4935  {
4936  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RNXT), m_evt_handler->m_curr->pos);
4937  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
4938  const char first = rem.str[0];
4939  _c4dbgpf("mapjson[RNXT]: '{}'", first);
4940  switch(first)
4941  {
4942  case ',':
4943  {
4944  _c4dbgp("seqjson[RNXT]: expect next val");
4945  addrem_flags(RVAL, RNXT);
4946  m_evt_handler->add_sibling();
4947  _line_progressed(1);
4948  break;
4949  }
4950  case ']':
4951  {
4952  _c4dbgp("seqjson[RNXT]: end!");
4953  _end_seq_flow();
4954  _line_progressed(1);
4955  goto seqjson_finish;
4956  }
4957  default:
4958  _c4err("parse error");
4959  }
4960  }
4961 
4962  seqjson_again:
4963  _c4dbgt("seqjson: go again", 0);
4964  if(_finished_line())
4965  {
4966  if C4_LIKELY(!_finished_file())
4967  {
4968  _line_ended();
4969  _scan_line();
4970  _c4dbgnextline();
4971  }
4972  else
4973  {
4974  _c4err("missing terminating ]");
4975  }
4976  }
4977  goto seqjson_start;
4978 
4979  seqjson_finish:
4980  _c4dbgp("seqjson: finish");
4981 }
4982 
4983 
4984 //-----------------------------------------------------------------------------
4985 
4986 template<class EventHandler>
4987 void ParseEngine<EventHandler>::_handle_map_json()
4988 {
4989 mapjson_start:
4990  _c4dbgpf("handle2_map_json: node_id={} level={} indentation={}", m_evt_handler->m_curr->node_id, m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref);
4991 
4992  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RMAP), m_evt_handler->m_curr->pos);
4993  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RFLOW), m_evt_handler->m_curr->pos);
4994  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
4995  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RKEY|RKCL|RVAL|RNXT), m_evt_handler->m_curr->pos);
4996  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, 1 == (has_any(RKEY) + has_any(RKCL) + has_any(RVAL) + has_any(RNXT)), m_evt_handler->m_curr->pos);
4997 
4998  _handle_flow_skip_whitespace();
4999  csubstr rem = m_evt_handler->m_curr->line_contents.rem;
5000  if(!rem.len)
5001  goto mapjson_again;
5002 
5003  if(has_any(RKEY))
5004  {
5005  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
5006  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
5007  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
5008  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
5009  const char first = rem.str[0];
5010  _c4dbgpf("mapjson[RKEY]: '{}'", first);
5011  switch(first)
5012  {
5013  case '"':
5014  {
5015  _c4dbgp("mapjson[RKEY]: scanning double-quoted scalar");
5016  ScannedScalar sc = _scan_scalar_dquot();
5017  csubstr maybe_filtered = _maybe_filter_key_scalar_dquot(sc);
5018  m_evt_handler->set_key_scalar_dquoted(maybe_filtered);
5019  addrem_flags(RKCL, RKEY);
5020  break;
5021  }
5022  case '}': // this happens on a trailing comma like ", }"
5023  {
5024  _c4dbgp("mapjson[RKEY]: end!");
5025  _end_map_flow();
5026  _line_progressed(1);
5027  goto mapjson_finish;
5028  }
5029  default:
5030  _c4err("parse error");
5031  }
5032  }
5033  else if(has_any(RVAL))
5034  {
5035  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
5036  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
5037  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
5038  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
5039  const char first = rem.str[0];
5040  _c4dbgpf("mapjson[RVAL]: '{}'", first);
5041  switch(first)
5042  {
5043  case '"':
5044  {
5045  _c4dbgp("mapjson[RVAL]: scanning double-quoted scalar");
5046  ScannedScalar sc = _scan_scalar_dquot();
5047  csubstr maybe_filtered = _maybe_filter_val_scalar_dquot(sc);
5048  m_evt_handler->set_val_scalar_dquoted(maybe_filtered);
5049  addrem_flags(RNXT, RVAL);
5050  break;
5051  }
5052  case '[':
5053  {
5054  _c4dbgp("mapjson[RVAL]: start val seqjson");
5055  addrem_flags(RNXT, RVAL);
5056  m_evt_handler->begin_seq_val_flow();
5057  _set_indentation(m_evt_handler->m_parent->indref);
5058  addrem_flags(RSEQ|RVAL, RMAP|RNXT);
5059  _line_progressed(1);
5060  goto mapjson_finish;
5061  }
5062  case '{':
5063  {
5064  _c4dbgp("mapjson[RVAL]: start val mapjson");
5065  addrem_flags(RNXT, RVAL);
5066  m_evt_handler->begin_map_val_flow();
5067  _set_indentation(m_evt_handler->m_parent->indref);
5068  addrem_flags(RKEY, RNXT);
5069  _line_progressed(1);
5070  // keep going in this function
5071  break;
5072  }
5073  default:
5074  {
5075  ScannedScalar sc;
5076  if(_scan_scalar_map_json(&sc))
5077  {
5078  _c4dbgp("mapjson[RVAL]: plain scalar.");
5079  csubstr maybe_filtered = _maybe_filter_val_scalar_plain(sc, m_evt_handler->m_curr->indref);
5080  m_evt_handler->set_val_scalar_plain(maybe_filtered);
5081  addrem_flags(RNXT, RVAL);
5082  }
5083  else
5084  {
5085  _c4err("parse error");
5086  }
5087  break;
5088  }
5089  }
5090  }
5091  else if(has_any(RKCL)) // read the key colon
5092  {
5093  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
5094  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
5095  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
5096  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
5097  const char first = rem.str[0];
5098  _c4dbgpf("mapjson[RKCL]: '{}'", first);
5099  if(first == ':')
5100  {
5101  _c4dbgp("mapjson[RKCL]: found the colon");
5102  addrem_flags(RVAL, RKCL);
5103  _line_progressed(1);
5104  }
5105  else
5106  {
5107  _c4err("parse error");
5108  }
5109  }
5110  else if(has_any(RNXT))
5111  {
5112  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
5113  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
5114  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
5115  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
5116  _c4dbgpf("mapjson[RNXT]: '{}'", rem.str[0]);
5117  if(rem.begins_with(','))
5118  {
5119  _c4dbgp("mapjson[RNXT]: expect next keyval");
5120  m_evt_handler->add_sibling();
5121  addrem_flags(RKEY, RNXT);
5122  _line_progressed(1);
5123  }
5124  else if(rem.begins_with('}'))
5125  {
5126  _c4dbgp("mapjson[RNXT]: end!");
5127  _end_map_flow();
5128  _line_progressed(1);
5129  goto mapjson_finish;
5130  }
5131  else
5132  {
5133  _c4err("parse error"); // LCOV_EXCL_LINE
5134  }
5135  }
5136 
5137  mapjson_again:
5138  _c4dbgt("mapjson: go again", 0);
5139  if(_finished_line())
5140  {
5141  if C4_LIKELY(!_finished_file())
5142  {
5143  _line_ended();
5144  _scan_line();
5145  _c4dbgnextline();
5146  }
5147  else
5148  {
5149  _c4err("missing terminating }");
5150  }
5151  }
5152  goto mapjson_start;
5153 
5154  mapjson_finish:
5155  _c4dbgp("mapjson: finish");
5156 }
5157 
5158 
5159 //-----------------------------------------------------------------------------
5160 
5161 template<class EventHandler>
5162 void ParseEngine<EventHandler>::_handle_seq_imap()
5163 {
5164 seqimap_start:
5165  _c4dbgpf("handle2_seq_imap: node_id={} level={} indref={}", m_evt_handler->m_curr->node_id, m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref);
5166 
5167  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RSEQIMAP), m_evt_handler->m_curr->pos);
5168  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
5169  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RVAL|RNXT|QMRK|RKCL), m_evt_handler->m_curr->pos);
5170  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, 1 == has_all(RVAL) + has_all(RNXT) + has_all(QMRK) + has_all(RKCL), m_evt_handler->m_curr->pos);
5171  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_stack.size() >= 3, m_evt_handler->m_curr->pos);
5172 
5173  _handle_flow_skip_whitespace();
5174  csubstr rem = m_evt_handler->m_curr->line_contents.rem;
5175  if(!rem.len)
5176  goto seqimap_again;
5177 
5178  if(has_any(RVAL))
5179  {
5180  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RVAL), m_evt_handler->m_curr->pos);
5181  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
5182  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
5183  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
5184  const char first = rem.str[0];
5185  _c4dbgpf("seqimap[RVAL]: '{}'", _c4prc(first));
5186  ScannedScalar sc;
5187  if(first == '\'')
5188  {
5189  _c4dbgp("seqimap[RVAL]: scanning single-quoted scalar");
5190  sc = _scan_scalar_squot();
5191  csubstr maybe_filtered = _maybe_filter_val_scalar_squot(sc);
5192  _handle_annotations_before_blck_val_scalar();
5193  m_evt_handler->set_val_scalar_squoted(maybe_filtered);
5194  _end_map_flow();
5195  goto seqimap_finish;
5196  }
5197  else if(first == '"')
5198  {
5199  _c4dbgp("seqimap[RVAL]: scanning double-quoted scalar");
5200  sc = _scan_scalar_dquot();
5201  csubstr maybe_filtered = _maybe_filter_val_scalar_dquot(sc);
5202  _handle_annotations_before_blck_val_scalar();
5203  m_evt_handler->set_val_scalar_dquoted(maybe_filtered);
5204  _end_map_flow();
5205  goto seqimap_finish;
5206  }
5207  // block scalars (ie | and >) cannot appear in flow containers
5208  else if(_scan_scalar_plain_map_flow(&sc))
5209  {
5210  _c4dbgp("seqimap[RVAL]: it's a scalar.");
5211  csubstr maybe_filtered = _maybe_filter_val_scalar_plain(sc, m_evt_handler->m_curr->indref);
5212  _handle_annotations_before_blck_val_scalar();
5213  m_evt_handler->set_val_scalar_plain(maybe_filtered);
5214  _end_map_flow();
5215  goto seqimap_finish;
5216  }
5217  else if(first == '[')
5218  {
5219  _c4dbgp("seqimap[RVAL]: start child seqflow");
5220  addrem_flags(RNXT, RVAL);
5221  _handle_annotations_before_blck_val_scalar();
5222  m_evt_handler->begin_seq_val_flow();
5223  addrem_flags(RVAL, RNXT|RSEQIMAP);
5224  _set_indentation(m_evt_handler->m_parent->indref);
5225  _line_progressed(1);
5226  goto seqimap_finish;
5227  }
5228  else if(first == '{')
5229  {
5230  _c4dbgp("seqimap[RVAL]: start child mapflow");
5231  addrem_flags(RNXT, RVAL);
5232  _handle_annotations_before_blck_val_scalar();
5233  m_evt_handler->begin_map_val_flow();
5234  addrem_flags(RMAP|RKEY, RSEQ|RVAL|RSEQIMAP|RNXT);
5235  _set_indentation(m_evt_handler->m_parent->indref);
5236  _line_progressed(1);
5237  goto seqimap_finish;
5238  }
5239  else if(first == ',' || first == ']')
5240  {
5241  _c4dbgp("seqimap[RVAL]: finish without val.");
5242  _handle_annotations_before_blck_val_scalar();
5243  m_evt_handler->set_val_scalar_plain_empty();
5244  _end_map_flow();
5245  goto seqimap_finish;
5246  }
5247  else if(first == '*')
5248  {
5249  csubstr ref = _scan_ref_seq();
5250  _c4dbgpf("seqimap[RVAL]: ref! {}", prs_(ref));
5251  _handle_valref(ref);
5252  addrem_flags(RNXT, RVAL);
5253  }
5254  else if(first == '&')
5255  {
5256  csubstr anchor = _scan_anchor();
5257  _c4dbgpf("seqimap[RVAL]: anchor! {}", prs_(anchor));
5258  _add_annotation(&m_pending_anchors, anchor);
5259  }
5260  else if(first == '!')
5261  {
5262  csubstr tag = _scan_tag();
5263  _c4dbgpf("seqimap[RVAL]: tag! {}", prs_(tag));
5264  _add_annotation(&m_pending_tags, tag);
5265  }
5266  else
5267  {
5268  _c4err("parse error"); // LCOV_EXCL_LINE
5269  }
5270  }
5271  else if(has_any(RNXT))
5272  {
5273  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RNXT), m_evt_handler->m_curr->pos);
5274  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
5275  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
5276  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
5277  const char first = rem.str[0];
5278  _c4dbgpf("seqimap[RNXT]: '{}'", _c4prc(first));
5279  if(first == ',' || first == ']')
5280  {
5281  // we may get here because a map or a seq started and we
5282  // return later
5283  _c4dbgp("seqimap: done");
5284  _end_map_flow();
5285  goto seqimap_finish;
5286  }
5287  else
5288  {
5289  _c4err("parse error"); // LCOV_EXCL_LINE
5290  }
5291  }
5292  else if(has_any(QMRK))
5293  {
5294  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(QMRK), m_evt_handler->m_curr->pos);
5295  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
5296  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
5297  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
5298  const char first = rem.str[0];
5299  _c4dbgpf("seqimap[QMRK]: '{}'", _c4prc(first));
5300  ScannedScalar sc;
5301  if(first == '\'')
5302  {
5303  _c4dbgp("seqimap[QMRK]: scanning single-quoted scalar");
5304  sc = _scan_scalar_squot();
5305  csubstr maybe_filtered = _maybe_filter_key_scalar_squot(sc);
5306  m_evt_handler->set_key_scalar_squoted(maybe_filtered);
5307  addrem_flags(RKCL, QMRK);
5308  goto seqimap_again;
5309  }
5310  else if(first == '"')
5311  {
5312  _c4dbgp("seqimap[QMRK]: scanning double-quoted scalar");
5313  sc = _scan_scalar_dquot();
5314  csubstr maybe_filtered = _maybe_filter_key_scalar_dquot(sc);
5315  m_evt_handler->set_key_scalar_dquoted(maybe_filtered);
5316  addrem_flags(RKCL, QMRK);
5317  goto seqimap_again;
5318  }
5319  // block scalars (ie | and >) cannot appear in flow containers
5320  else if(_scan_scalar_plain_map_flow(&sc))
5321  {
5322  _c4dbgp("seqimap[QMRK]: it's a scalar.");
5323  csubstr maybe_filtered = _maybe_filter_key_scalar_plain(sc, m_evt_handler->m_curr->indref);
5324  m_evt_handler->set_key_scalar_plain(maybe_filtered);
5325  addrem_flags(RKCL, QMRK);
5326  goto seqimap_again;
5327  }
5328  else if(first == '[')
5329  {
5330  _c4dbgp("seqimap[QMRK]: start child seqflow");
5331  addrem_flags(RKCL, QMRK);
5332  m_evt_handler->begin_seq_key_flow();
5333  addrem_flags(RSEQ|RVAL, RKCL|RSEQIMAP);
5334  _set_indentation(m_evt_handler->m_parent->indref);
5335  _line_progressed(1);
5336  goto seqimap_finish;
5337  }
5338  else if(first == '{')
5339  {
5340  _c4dbgp("seqimap[QMRK]: start child mapflow");
5341  addrem_flags(RKCL, QMRK);
5342  m_evt_handler->begin_map_key_flow();
5343  addrem_flags(RMAP|RKEY, RSEQ|RKCL|RSEQIMAP);
5344  _set_indentation(m_evt_handler->m_parent->indref);
5345  _line_progressed(1);
5346  goto seqimap_finish;
5347  }
5348  else if(first == ',' || first == ']')
5349  {
5350  _c4dbgp("seqimap[QMRK]: finish without key.");
5351  m_evt_handler->set_key_scalar_plain_empty();
5352  m_evt_handler->set_val_scalar_plain_empty();
5353  _end_map_flow();
5354  goto seqimap_finish;
5355  }
5356  else if(first == '&')
5357  {
5358  csubstr anchor = _scan_anchor();
5359  _c4dbgp("seqimap[QMRK]: anchor!");
5360  m_evt_handler->set_key_anchor(anchor);
5361  }
5362  else if(first == '*')
5363  {
5364  csubstr ref = _scan_ref_seq();
5365  _c4dbgp("seqimap[QMRK]: ref!");
5366  _handle_keyref(ref);
5367  addrem_flags(RKCL, QMRK);
5368  }
5369  else
5370  {
5371  _c4err("parse error"); // LCOV_EXCL_LINE
5372  }
5373  }
5374  else if(has_any(RKCL))
5375  {
5376  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
5377  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
5378  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
5379  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RKCL), m_evt_handler->m_curr->pos);
5380  const char first = rem.str[0];
5381  _c4dbgpf("seqimap[RKCL]: '{}'", _c4prc(first));
5382  if(first == ':')
5383  {
5384  _c4dbgp("seqimap[RKCL]: found ':'");
5385  addrem_flags(RVAL, RKCL);
5386  _line_progressed(1);
5387  goto seqimap_again;
5388  }
5389  else if(first == ',' || first == ']')
5390  {
5391  _c4dbgp("seqimap[RKCL]: found ','. finish without val");
5392  m_evt_handler->set_val_scalar_plain_empty();
5393  _end_map_flow();
5394  goto seqimap_finish;
5395  }
5396  else
5397  {
5398  _c4err("parse error"); // LCOV_EXCL_LINE
5399  }
5400  }
5401 
5402  seqimap_again:
5403  _c4dbgt("seqimap: go again", 0);
5404  if(_finished_line())
5405  {
5406  if C4_LIKELY(!_finished_file())
5407  {
5408  _line_ended();
5409  _scan_line();
5410  _c4dbgnextline();
5411  }
5412  else
5413  {
5414  _c4err("parse error");
5415  }
5416  }
5417  goto seqimap_start;
5418 
5419  seqimap_finish:
5420  _c4dbgp("seqimap: finish");
5421 }
5422 
5423 
5424 //-----------------------------------------------------------------------------
5425 
5426 template<class EventHandler>
5427 void ParseEngine<EventHandler>::_handle_seq_flow()
5428 {
5429 seqflow_start:
5430  _c4dbgpf("handle_seq_flow: node_id={} level={} indentation={}", m_evt_handler->m_curr->node_id, m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref);
5431 
5432  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
5433  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RSEQ), m_evt_handler->m_curr->pos);
5434  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RFLOW), m_evt_handler->m_curr->pos);
5435  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RVAL|RNXT), m_evt_handler->m_curr->pos);
5436  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RVAL) != has_all(RNXT), m_evt_handler->m_curr->pos);
5437  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->indref != npos, m_evt_handler->m_curr->pos);
5438 
5439  if(m_evt_handler->m_curr->at_line_beginning())
5440  {
5441  _handle_flow_line_beginning();
5442  }
5443 
5444  _handle_flow_skip_whitespace();
5445  if(!m_evt_handler->m_curr->line_contents.rem.len)
5446  goto seqflow_again;
5447 
5448  if(has_any(RVAL))
5449  {
5450  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
5451  const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
5452  ScannedScalar sc;
5453  if(first == '\'')
5454  {
5455  _c4dbgp("seqflow[RVAL]: scanning single-quoted scalar");
5456  sc = _scan_scalar_squot();
5457  csubstr maybe_filtered = _maybe_filter_val_scalar_squot(sc);
5458  _handle_annotations_before_blck_val_scalar();
5459  m_evt_handler->set_val_scalar_squoted(maybe_filtered);
5460  addrem_flags(RNXT, RVAL);
5461  _mark_seqflow_val_end();
5462  }
5463  else if(first == '"')
5464  {
5465  _c4dbgp("seqflow[RVAL]: scanning double-quoted scalar");
5466  sc = _scan_scalar_dquot();
5467  csubstr maybe_filtered = _maybe_filter_val_scalar_dquot(sc);
5468  _handle_annotations_before_blck_val_scalar();
5469  m_evt_handler->set_val_scalar_dquoted(maybe_filtered);
5470  addrem_flags(RNXT, RVAL);
5471  _mark_seqflow_val_end();
5472  }
5473  // block scalars (ie | and >) cannot appear in flow containers
5474  else if(_scan_scalar_plain_seq_flow(&sc))
5475  {
5476  _c4dbgp("seqflow[RVAL]: it's a scalar.");
5477  csubstr maybe_filtered = _maybe_filter_val_scalar_plain(sc, m_evt_handler->m_curr->indref);
5478  _handle_annotations_before_blck_val_scalar();
5479  m_evt_handler->set_val_scalar_plain(maybe_filtered);
5480  addrem_flags(RNXT, RVAL);
5481  _mark_seqflow_val_end();
5482  }
5483  else if(first == '[')
5484  {
5485  _c4dbgp("seqflow[RVAL]: start child seqflow");
5486  addrem_flags(RNXT, RVAL);
5487  _handle_annotations_before_blck_val_scalar();
5488  m_evt_handler->begin_seq_val_flow();
5489  _set_indentation(m_evt_handler->m_parent->indref);
5490  addrem_flags(RVAL, RNXT);
5491  _line_progressed(1);
5492  }
5493  else if(first == '{')
5494  {
5495  _c4dbgp("seqflow[RVAL]: start child mapflow");
5496  addrem_flags(RNXT, RVAL);
5497  _handle_annotations_before_blck_val_scalar();
5498  m_evt_handler->begin_map_val_flow();
5499  _set_indentation(m_evt_handler->m_parent->indref);
5500  addrem_flags(RMAP|RKEY, RSEQ|RVAL|RNXT);
5501  _line_progressed(1);
5502  goto seqflow_finish;
5503  }
5504  else if(first == ']') // this happens on cases such as [] or [.., ]
5505  {
5506  _c4dbgp("seqflow[RVAL]: end!");
5507  if(m_pending_anchors.num_entries | m_pending_tags.num_entries)
5508  {
5509  _c4dbgp("seqflow[RVAL]: add pending annotations");
5510  _handle_annotations_before_blck_val_scalar();
5511  m_evt_handler->set_val_scalar_plain_empty();
5512  }
5513  _line_progressed(1);
5514  _end_seq_flow();
5515  goto seqflow_finish;
5516  }
5517  else if(first == '*')
5518  {
5519  csubstr ref = _scan_ref_seq();
5520  _c4dbgpf("seqflow[RVAL]: ref! {}", prs_(ref));
5521  _handle_valref(ref);
5522  addrem_flags(RNXT, RVAL);
5523  }
5524  else if(first == '&')
5525  {
5526  csubstr anchor = _scan_anchor();
5527  _c4dbgpf("seqflow[RVAL]: anchor! {}", prs_(anchor));
5528  _add_annotation(&m_pending_anchors, anchor);
5529  }
5530  else if(first == '!')
5531  {
5532  csubstr tag = _scan_tag();
5533  _c4dbgpf("seqflow[RVAL]: tag! {}", prs_(tag));
5534  _add_annotation(&m_pending_tags, tag);
5535  }
5536  else if(first == ':')
5537  {
5538  _c4dbgpf("seqflow[RVAL]: actually seqimap at node[{}], with empty key", m_evt_handler->m_curr->node_id);
5539  addrem_flags(RNXT, RVAL);
5540  m_evt_handler->begin_map_val_flow();
5541  _set_indentation(m_evt_handler->m_parent->indref);
5542  _handle_annotations_before_blck_key_scalar();
5543  m_evt_handler->set_key_scalar_plain_empty();
5544  addrem_flags(RSEQIMAP|RVAL, RSEQ|RNXT);
5545  _line_progressed(1);
5546  goto seqflow_finish;
5547  }
5548  else if(first == '?')
5549  {
5550  _c4dbgp("seqflow[RVAL]: start child mapflow, explicit key");
5551  addrem_flags(RNXT, RVAL);
5552  m_evt_handler->begin_map_val_flow();
5553  _set_indentation(m_evt_handler->m_parent->indref);
5554  addrem_flags(RSEQIMAP|QMRK, RSEQ|RNXT);
5555  _line_progressed(1);
5556  _maybe_skip_whitespace_tokens();
5557  goto seqflow_finish;
5558  }
5559  else if(first == ',')
5560  {
5561  if(m_pending_anchors.num_entries || m_pending_tags.num_entries)
5562  {
5563  _c4dbgp("seqflow[RVAL]: add pending annotations");
5564  _handle_annotations_before_blck_val_scalar();
5565  m_evt_handler->set_val_scalar_plain_empty();
5566  addrem_flags(RNXT, RVAL);
5567  _mark_seqflow_val_end();
5568  }
5569  else
5570  {
5571  _c4err("parse error");
5572  }
5573  }
5574  else
5575  {
5576  _c4err("parse error");
5577  }
5578  }
5579  else // RNXT
5580  {
5581  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RNXT), m_evt_handler->m_curr->pos);
5582  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
5583  const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
5584  if(first == ',')
5585  {
5586  _c4dbgp("seqflow[RNXT]: expect next val");
5587  addrem_flags(RVAL, RNXT);
5588  m_evt_handler->add_sibling();
5589  _line_progressed(1);
5590  if(m_evt_handler->m_curr->line_contents.rem.begins_with('#'))
5591  {
5592  _c4err("parse error: invalid comment after comma");
5593  }
5594  _mark_seqflow_val_end();
5595  }
5596  else if(first == ']')
5597  {
5598  _c4dbgp("seqflow[RNXT]: end!");
5599  _line_progressed(1);
5600  _end_seq_flow();
5601  goto seqflow_finish;
5602  }
5603  else if(first == ':')
5604  {
5605  _c4dbgpf("seqflow[RNXT]: line@valend={} line@now={}", m_prev_val_end, m_evt_handler->m_curr->pos.line);
5606  if(m_prev_val_end != NONE && m_evt_handler->m_curr->pos.line == m_prev_val_end)
5607  {
5608  _c4dbgpf("seqflow[RNXT]: actually seqimap at node[{}]", m_evt_handler->m_curr->node_id);
5609  m_evt_handler->actually_val_is_first_key_of_new_map_flow();
5610  _set_indentation(m_evt_handler->m_parent->indref);
5611  _line_progressed(1);
5612  addrem_flags(RSEQIMAP|RVAL, RNXT);
5613  goto seqflow_finish;
5614  }
5615  else
5616  {
5617  _c4err("parse error");
5618  }
5619  }
5620  else
5621  {
5622  _c4err("parse error");
5623  }
5624  }
5625 
5626  seqflow_again:
5627  _c4dbgt("seqflow: go again", 0);
5628  if(_finished_line())
5629  {
5630  if C4_LIKELY(!_finished_file())
5631  {
5632  _line_ended();
5633  _scan_line();
5634  _c4dbgnextline();
5635  }
5636  else
5637  {
5638  _c4err("missing terminating ]");
5639  }
5640  }
5641  goto seqflow_start;
5642 
5643  seqflow_finish:
5644  _c4dbgp("seqflow: finish");
5645 }
5646 
5647 
5648 //-----------------------------------------------------------------------------
5649 
5650 template<class EventHandler>
5651 void ParseEngine<EventHandler>::_handle_map_flow()
5652 {
5653 mapflow_start:
5654  _c4dbgpf("handle_map_flow: node_id={} level={} indentation={}", m_evt_handler->m_curr->node_id, m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref);
5655 
5656  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RMAP), m_evt_handler->m_curr->pos);
5657  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RFLOW), m_evt_handler->m_curr->pos);
5658  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RKEY|RKCL|RVAL|RNXT|QMRK), m_evt_handler->m_curr->pos);
5659  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, 1 == (has_any(RKEY) + has_any(RKCL) + has_any(RVAL) + has_any(RNXT) + has_any(QMRK)), m_evt_handler->m_curr->pos);
5660 
5661  if(m_evt_handler->m_curr->at_line_beginning())
5662  {
5663  _handle_flow_line_beginning();
5664  }
5665 
5666  _handle_flow_skip_whitespace();
5667  if(!m_evt_handler->m_curr->line_contents.rem.len)
5668  goto mapflow_again;
5669 
5670  if(has_any(RKEY))
5671  {
5672  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
5673  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
5674  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
5675  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
5676  const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
5677  _c4dbgpf("mapflow[RKEY]: '{}'", first);
5678  ScannedScalar sc;
5679  if(first == '\'')
5680  {
5681  _c4dbgp("mapflow[RKEY]: scanning single-quoted scalar");
5682  sc = _scan_scalar_squot();
5683  csubstr maybe_filtered = _maybe_filter_key_scalar_squot(sc);
5684  _handle_annotations_before_blck_key_scalar();
5685  m_evt_handler->set_key_scalar_squoted(maybe_filtered);
5686  addrem_flags(RKCL, RKEY|QMRK);
5687  }
5688  else if(first == '"')
5689  {
5690  _c4dbgp("mapflow[RKEY]: scanning double-quoted scalar");
5691  sc = _scan_scalar_dquot();
5692  csubstr maybe_filtered = _maybe_filter_key_scalar_dquot(sc);
5693  _handle_annotations_before_blck_key_scalar();
5694  m_evt_handler->set_key_scalar_dquoted(maybe_filtered);
5695  addrem_flags(RKCL, RKEY|QMRK);
5696  }
5697  // block scalars (ie | and >) cannot appear in flow containers
5698  else if(_scan_scalar_plain_map_flow(&sc))
5699  {
5700  _c4dbgp("mapflow[RKEY]: plain scalar");
5701  csubstr maybe_filtered = _maybe_filter_key_scalar_plain(sc, m_evt_handler->m_curr->indref);
5702  _handle_annotations_before_blck_key_scalar();
5703  m_evt_handler->set_key_scalar_plain(maybe_filtered);
5704  addrem_flags(RKCL, RKEY|QMRK);
5705  }
5706  else if(first == '?')
5707  {
5708  _c4dbgp("mapflow[RKEY]: explicit key");
5709  _handle_annotations_before_blck_key_scalar();
5710  addrem_flags(QMRK, RKEY);
5711  _line_progressed(1);
5712  _maybe_skip_whitespace_tokens();
5713  }
5714  else if(first == ':')
5715  {
5716  _c4dbgp("mapflow[RKEY]: setting empty key");
5717  _handle_annotations_before_blck_key_scalar();
5718  m_evt_handler->set_key_scalar_plain_empty();
5719  addrem_flags(RVAL, RKEY|QMRK);
5720  _line_progressed(1);
5721  _maybe_skip_whitespace_tokens();
5722  }
5723  else if(first == ',')
5724  {
5725  _c4dbgp("mapflow[RKEY]: comma!");
5726  if(!_handle_annotations_before_unexpected_flow_token_rkey())
5727  _c4err("unexpected comma");
5728  addrem_flags(RNXT, RKEY|QMRK);
5729  // keep going in this function
5730  }
5731  else if(first == '}') // this happens on a trailing comma like ", }"
5732  {
5733  _c4dbgp("mapflow[RKEY]: end!");
5734  (void)_handle_annotations_before_unexpected_flow_token_rkey();
5735  _line_progressed(1);
5736  _end_map_flow();
5737  goto mapflow_finish;
5738  }
5739  else if(first == '&')
5740  {
5741  csubstr anchor = _scan_anchor();
5742  _c4dbgpf("mapflow[RKEY]: key anchor! {}", prs_(anchor));
5743  _add_annotation(&m_pending_anchors, anchor);
5744  }
5745  else if(first == '!')
5746  {
5747  csubstr tag = _scan_tag();
5748  _c4dbgpf("mapflow[RKEY]: tag! {}", prs_(tag));
5749  _add_annotation(&m_pending_tags, tag);
5750  }
5751  else if(first == '*')
5752  {
5753  csubstr ref = _scan_ref_map();
5754  _c4dbgpf("mapflow[RKEY]: key ref! {}", prs_(ref));
5755  _handle_keyref(ref);
5756  addrem_flags(RKCL, RKEY);
5757  }
5758  else if(first == '[')
5759  {
5760  // RYML's tree cannot store container keys, but that's
5761  // handled inside the tree event handler. Other handler
5762  // types may be able to handle it.
5763  _c4dbgp("mapflow[RKEY]: start child seqflow (!)");
5764  _handle_annotations_before_blck_key_scalar();
5765  addrem_flags(RKCL, RKEY);
5766  m_evt_handler->begin_seq_key_flow();
5767  addrem_flags(RSEQ|RVAL, RMAP|RKCL);
5768  _set_indentation(m_evt_handler->m_parent->indref);
5769  _line_progressed(1);
5770  goto mapflow_finish;
5771  }
5772  else if(first == '{')
5773  {
5774  // RYML's tree cannot store container keys, but that's
5775  // handled inside the tree event handler. Other handler
5776  // types may be able to handle it.
5777  _c4dbgp("mapflow[RKEY]: start child mapflow (!)");
5778  _handle_annotations_before_blck_key_scalar();
5779  addrem_flags(RKCL, RKEY);
5780  m_evt_handler->begin_map_key_flow();
5781  addrem_flags(RKEY, RVAL|RKCL);
5782  _set_indentation(m_evt_handler->m_parent->indref);
5783  _line_progressed(1);
5784  // keep going in this function
5785  }
5786  else
5787  {
5788  _c4err("parse error"); // LCOV_EXCL_LINE
5789  }
5790  }
5791  else if(has_any(RKCL)) // read the key colon
5792  {
5793  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
5794  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
5795  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
5796  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
5797  const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
5798  _c4dbgpf("mapflow[RKCL]: '{}'", first);
5799  if(first == ':')
5800  {
5801  _c4dbgp("mapflow[RKCL]: found the colon");
5802  addrem_flags(RVAL, RKCL);
5803  _line_progressed(1);
5804  }
5805  else if(first == '}')
5806  {
5807  _c4dbgp("mapflow[RKCL]: end with missing val!");
5808  addrem_flags(RVAL, RKCL);
5809  m_evt_handler->set_val_scalar_plain_empty();
5810  _line_progressed(1);
5811  _end_map_flow();
5812  goto mapflow_finish;
5813  }
5814  else if(first == ',')
5815  {
5816  _c4dbgp("mapflow[RKCL]: got comma. val is missing");
5817  m_evt_handler->set_val_scalar_plain_empty();
5818  m_evt_handler->add_sibling();
5819  addrem_flags(RKEY, RKCL);
5820  _line_progressed(1);
5821  if(m_evt_handler->m_curr->line_contents.rem.begins_with('#'))
5822  {
5823  _c4err("parse error: invalid comment after comma");
5824  }
5825  }
5826  else
5827  {
5828  _c4err("parse error");
5829  }
5830  }
5831  else if(has_any(RVAL))
5832  {
5833  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
5834  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
5835  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
5836  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
5837  const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
5838  _c4dbgpf("mapflow[RVAL]: '{}'", first);
5839  ScannedScalar sc;
5840  if(first == '\'')
5841  {
5842  _c4dbgp("mapflow[RVAL]: scanning single-quoted scalar");
5843  sc = _scan_scalar_squot();
5844  csubstr maybe_filtered = _maybe_filter_val_scalar_squot(sc);
5845  _handle_annotations_before_blck_val_scalar();
5846  m_evt_handler->set_val_scalar_squoted(maybe_filtered);
5847  addrem_flags(RNXT, RVAL);
5848  }
5849  else if(first == '"')
5850  {
5851  _c4dbgp("mapflow[RVAL]: scanning double-quoted scalar");
5852  sc = _scan_scalar_dquot();
5853  csubstr maybe_filtered = _maybe_filter_val_scalar_dquot(sc);
5854  _handle_annotations_before_blck_val_scalar();
5855  m_evt_handler->set_val_scalar_dquoted(maybe_filtered);
5856  addrem_flags(RNXT, RVAL);
5857  }
5858  // block scalars (ie | and >) cannot appear in flow containers
5859  else if(_scan_scalar_plain_map_flow(&sc))
5860  {
5861  _c4dbgp("mapflow[RVAL]: plain scalar.");
5862  csubstr maybe_filtered = _maybe_filter_val_scalar_plain(sc, m_evt_handler->m_curr->indref);
5863  _handle_annotations_before_blck_val_scalar();
5864  m_evt_handler->set_val_scalar_plain(maybe_filtered);
5865  addrem_flags(RNXT, RVAL);
5866  }
5867  else if(first == '[')
5868  {
5869  _c4dbgp("mapflow[RVAL]: start val seqflow");
5870  addrem_flags(RNXT, RVAL);
5871  _handle_annotations_before_blck_val_scalar();
5872  m_evt_handler->begin_seq_val_flow();
5873  _set_indentation(m_evt_handler->m_parent->indref);
5874  addrem_flags(RSEQ|RVAL, RMAP|RNXT);
5875  _line_progressed(1);
5876  goto mapflow_finish;
5877  }
5878  else if(first == '{')
5879  {
5880  _c4dbgp("mapflow[RVAL]: start val mapflow");
5881  addrem_flags(RNXT, RVAL);
5882  _handle_annotations_before_blck_val_scalar();
5883  m_evt_handler->begin_map_val_flow();
5884  _set_indentation(m_evt_handler->m_parent->indref);
5885  addrem_flags(RKEY, RNXT);
5886  _line_progressed(1);
5887  // keep going in this function
5888  }
5889  else if(first == '}')
5890  {
5891  _c4dbgp("mapflow[RVAL]: end!");
5892  _handle_annotations_before_blck_val_scalar();
5893  m_evt_handler->set_val_scalar_plain_empty();
5894  _line_progressed(1);
5895  _end_map_flow();
5896  goto mapflow_finish;
5897  }
5898  else if(first == ',')
5899  {
5900  _c4dbgp("mapflow[RVAL]: empty val!");
5901  _handle_annotations_before_blck_val_scalar();
5902  m_evt_handler->set_val_scalar_plain_empty();
5903  addrem_flags(RNXT, RVAL);
5904  // keep going in this function
5905  }
5906  else if(first == '*')
5907  {
5908  csubstr ref = _scan_ref_map();
5909  _c4dbgpf("mapflow[RVAL]: key ref! {}", prs_(ref));
5910  _handle_valref(ref);
5911  addrem_flags(RNXT, RVAL);
5912  }
5913  else if(first == '&')
5914  {
5915  csubstr anchor = _scan_anchor();
5916  _c4dbgpf("mapflow[RVAL]: key anchor! {}", prs_(anchor));
5917  _add_annotation(&m_pending_anchors, anchor);
5918  }
5919  else if(first == '!')
5920  {
5921  csubstr tag = _scan_tag();
5922  _c4dbgpf("mapflow[RVAL]: tag! {}", prs_(tag));
5923  _add_annotation(&m_pending_tags, tag);
5924  }
5925  else
5926  {
5927  _c4err("parse error");
5928  }
5929  }
5930  else if(has_any(RNXT))
5931  {
5932  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
5933  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
5934  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
5935  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
5936  _c4dbgpf("mapflow[RNXT]: '{}'", m_evt_handler->m_curr->line_contents.rem.str[0]);
5937  if(m_evt_handler->m_curr->line_contents.rem.begins_with(','))
5938  {
5939  _c4dbgp("mapflow[RNXT]: expect next keyval");
5940  m_evt_handler->add_sibling();
5941  addrem_flags(RKEY, RNXT);
5942  _line_progressed(1);
5943  if(m_evt_handler->m_curr->line_contents.rem.begins_with('#'))
5944  {
5945  _c4err("parse error: invalid comment after comma");
5946  }
5947  }
5948  else if(m_evt_handler->m_curr->line_contents.rem.begins_with('}'))
5949  {
5950  _c4dbgp("mapflow[RNXT]: end!");
5951  _line_progressed(1);
5952  _end_map_flow();
5953  goto mapflow_finish;
5954  }
5955  else
5956  {
5957  _c4err("parse error");
5958  }
5959  }
5960  else if(has_any(QMRK))
5961  {
5962  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
5963  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
5964  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
5965  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
5966  const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
5967  _c4dbgpf("mapflow[QMRK]: '{}'", first);
5968  ScannedScalar sc;
5969  if(first == '\'')
5970  {
5971  _c4dbgp("mapflow[QMRK]: scanning single-quoted scalar");
5972  sc = _scan_scalar_squot();
5973  csubstr maybe_filtered = _maybe_filter_key_scalar_squot(sc);
5974  _handle_annotations_before_blck_key_scalar();
5975  m_evt_handler->set_key_scalar_squoted(maybe_filtered);
5976  addrem_flags(RKCL, QMRK);
5977  }
5978  else if(first == '"')
5979  {
5980  _c4dbgp("mapflow[QMRK]: scanning double-quoted scalar");
5981  sc = _scan_scalar_dquot();
5982  csubstr maybe_filtered = _maybe_filter_key_scalar_dquot(sc);
5983  _handle_annotations_before_blck_key_scalar();
5984  m_evt_handler->set_key_scalar_dquoted(maybe_filtered);
5985  addrem_flags(RKCL, QMRK);
5986  }
5987  // block scalars (ie | and >) cannot appear in flow containers
5988  else if(_scan_scalar_plain_map_flow(&sc))
5989  {
5990  _c4dbgp("mapflow[QMRK]: plain scalar");
5991  csubstr maybe_filtered = _maybe_filter_key_scalar_plain(sc, m_evt_handler->m_curr->indref);
5992  _handle_annotations_before_blck_key_scalar();
5993  m_evt_handler->set_key_scalar_plain(maybe_filtered);
5994  addrem_flags(RKCL, QMRK);
5995  }
5996  else if(first == ':')
5997  {
5998  _c4dbgp("mapflow[QMRK]: setting empty key");
5999  _handle_annotations_before_blck_key_scalar();
6000  m_evt_handler->set_key_scalar_plain_empty();
6001  addrem_flags(RVAL, QMRK);
6002  _line_progressed(1);
6003  _maybe_skip_whitespace_tokens();
6004  }
6005  else if(first == '}') // this happens on a trailing comma like ", }"
6006  {
6007  _c4dbgp("mapflow[QMRK]: end!");
6008  _handle_annotations_before_blck_key_scalar();
6009  m_evt_handler->set_key_scalar_plain_empty();
6010  m_evt_handler->set_val_scalar_plain_empty();
6011  _end_map_flow();
6012  _line_progressed(1);
6013  goto mapflow_finish;
6014  }
6015  else if(first == ',')
6016  {
6017  _c4dbgp("mapflow[QMRK]: empty key+val!");
6018  _handle_annotations_before_blck_key_scalar();
6019  m_evt_handler->set_key_scalar_plain_empty();
6020  m_evt_handler->set_val_scalar_plain_empty();
6021  addrem_flags(RNXT, QMRK);
6022  }
6023  else if(first == '&')
6024  {
6025  csubstr anchor = _scan_anchor();
6026  _c4dbgpf("mapflow[QMRK]: key anchor! {}", prs_(anchor));
6027  _add_annotation(&m_pending_anchors, anchor);
6028  }
6029  else if(first == '*')
6030  {
6031  csubstr ref = _scan_ref_map();
6032  _c4dbgpf("mapflow[QMRK]: key ref! {}", prs_(ref));
6033  _handle_keyref(ref);
6034  addrem_flags(RKCL, QMRK);
6035  }
6036  else if(first == '[')
6037  {
6038  // RYML's tree cannot store container keys, but that's
6039  // handled inside the tree sink. Other sink types may be
6040  // able to handle it.
6041  _c4dbgp("mapflow[QMRK]: start child seqflow (!)");
6042  addrem_flags(RKCL, QMRK);
6043  _handle_annotations_before_blck_key_scalar();
6044  m_evt_handler->begin_seq_key_flow();
6045  addrem_flags(RSEQ|RVAL, RMAP|RKCL);
6046  _set_indentation(m_evt_handler->m_parent->indref);
6047  _line_progressed(1);
6048  goto mapflow_finish;
6049  }
6050  else if(first == '{')
6051  {
6052  // RYML's tree cannot store container keys, but that's
6053  // handled inside the tree sink. Other sink types may be
6054  // able to handle it.
6055  _c4dbgp("mapflow[QMRK]: start child mapflow (!)");
6056  addrem_flags(RKCL, QMRK);
6057  _handle_annotations_before_blck_key_scalar();
6058  m_evt_handler->begin_map_key_flow();
6059  _set_indentation(m_evt_handler->m_parent->indref);
6060  addrem_flags(RKEY, RKCL);
6061  _line_progressed(1);
6062  // keep going in this function
6063  }
6064  else if(first == '!')
6065  {
6066  csubstr tag = _scan_tag();
6067  _c4dbgpf("mapflow[QMRK]: tag! {}", prs_(tag));
6068  _add_annotation(&m_pending_tags, tag);
6069  }
6070  else
6071  {
6072  _c4err("parse error"); // LCOV_EXCL_LINE
6073  }
6074  }
6075 
6076  mapflow_again:
6077  _c4dbgt("mapflow: go again", 0);
6078  if(_finished_line())
6079  {
6080  if C4_LIKELY(!_finished_file())
6081  {
6082  _line_ended();
6083  _scan_line();
6084  _c4dbgnextline();
6085  }
6086  else
6087  {
6088  _c4err("missing terminating }");
6089  }
6090  }
6091  goto mapflow_start;
6092 
6093  mapflow_finish:
6094  _c4dbgp("mapflow: finish");
6095 }
6096 
6097 
6098 //-----------------------------------------------------------------------------
6099 
6100 template<class EventHandler>
6101 void ParseEngine<EventHandler>::_handle_seq_block()
6102 {
6103 seqblck_start:
6104  _c4dbgpf("handle_seq_block: seq_id={} node_id={} level={} indent={}", m_evt_handler->m_parent->node_id, m_evt_handler->m_curr->node_id, m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref);
6105 
6106  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RSEQ), m_evt_handler->m_curr->pos);
6107  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RBLCK), m_evt_handler->m_curr->pos);
6108  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RVAL|RNXT), m_evt_handler->m_curr->pos);
6109  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, 1 == (has_any(RVAL) + has_any(RNXT)), m_evt_handler->m_curr->pos);
6110 
6111  _maybe_skip_comment_strict();
6112  if(!m_evt_handler->m_curr->line_contents.rem.len)
6113  goto seqblck_again;
6114 
6115  if(has_any(RVAL))
6116  {
6117  _c4dbgpf("seqblck[RVAL]: col={}", m_evt_handler->m_curr->pos.col);
6118  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
6119  if(m_evt_handler->m_curr->at_line_beginning())
6120  {
6121  _c4dbgpf("seqblck[RVAL]: indref={} indentation={}", m_evt_handler->m_curr->indref+1, m_evt_handler->m_curr->line_contents.indentation);
6122  if(m_evt_handler->m_curr->indentation_ge_extra())
6123  {
6124  _c4dbgpf("seqblck[RVAL]: skip {} from indentation", m_evt_handler->m_curr->line_contents.indentation);
6125  _line_progressed(m_evt_handler->m_curr->line_contents.indentation);
6126  if(!m_evt_handler->m_curr->line_contents.rem.len)
6127  goto seqblck_again;
6128  }
6129  else if(m_evt_handler->m_curr->indentation_lt_extra())
6130  {
6131  _c4dbgp("seqblck[RVAL]: smaller indentation than RVAL!");
6132  if(m_evt_handler->m_curr->indentation_eq())
6133  {
6134  _c4dbgp("seqblck[RVAL]: smaller indentation than RVAL!");
6135  _handle_annotations_before_blck_val_scalar();
6136  m_evt_handler->set_val_scalar_plain_empty();
6137  addrem_flags(RNXT, RVAL);
6138  goto seqblck_again;
6139  }
6140  else
6141  {
6142  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->indentation_lt(), m_evt_handler->m_curr->pos);
6143  _c4dbgp("seqblck[RVAL]: smaller indentation!");
6144  _handle_indentation_pop_from_block_seq();
6145  goto seqblck_finish;
6146  }
6147  }
6148  else if(m_evt_handler->m_curr->line_contents.indentation == npos)
6149  {
6150  _c4dbgp("seqblck[RVAL]: empty line!");
6151  _line_progressed(m_evt_handler->m_curr->line_contents.rem.len);
6152  goto seqblck_again;
6153  }
6154  }
6155  RYML_ASSERT_PARSE_CB_(callbacks(), m_evt_handler->m_curr->line_contents.rem.len, m_evt_handler->m_curr->pos);
6156  const size_t startmark = _handle_block_skip_leading_whitespace();
6157  _c4dbgpf("seqblck[RVAL]: startmark={}", startmark);
6158  if(startmark == npos)
6159  {
6160  _c4dbgp("seqblck[RVAL]: whitespace only");
6161  goto seqblck_again;
6162  }
6163  const size_t tabmark = _handle_block_get_whitespace_mark();
6164  const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
6165  _c4dbgpf("seqblck[RVAL]: first='{}' currcol={}", first, m_evt_handler->m_curr->pos.col - 1);
6166  const size_t startline = m_evt_handler->m_curr->pos.line;
6167  _c4assert(m_evt_handler->m_curr->line_contents.current_col() >= m_bom_len);
6168  const size_t startindent = m_evt_handler->m_curr->line_contents.current_col() - m_bom_len;
6169  ScannedScalar sc;
6170  if(first == '\'')
6171  {
6172  _c4dbgp("seqblck[RVAL]: single-quoted scalar");
6173  sc = _scan_scalar_squot();
6174  if(!_maybe_scan_following_colon())
6175  {
6176  _c4dbgp("seqblck[RVAL]: set as val");
6177  _handle_annotations_before_blck_val_scalar();
6178  csubstr maybe_filtered = _maybe_filter_val_scalar_squot(sc); // VAL!
6179  m_evt_handler->set_val_scalar_squoted(maybe_filtered);
6180  addrem_flags(RNXT, RVAL);
6181  }
6182  else
6183  {
6184  _c4dbgp("seqblck[RVAL]: start mapblck, set scalar as key");
6185  _handle_block_check_leading_tabs(startmark);
6186  addrem_flags(RNXT, RVAL);
6187  _handle_annotations_before_start_mapblck(startline);
6188  _handle_colon();
6189  m_evt_handler->begin_map_val_block();
6190  _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
6191  csubstr maybe_filtered = _maybe_filter_key_scalar_squot(sc); // KEY!
6192  m_evt_handler->set_key_scalar_squoted(maybe_filtered);
6193  addrem_flags(RMAP|RVAL, RSEQ|RNXT);
6194  _maybe_skip_whitespace_tokens();
6195  goto seqblck_finish;
6196  }
6197  }
6198  else if(first == '"')
6199  {
6200  _c4dbgp("seqblck[RVAL]: double-quoted scalar");
6201  sc = _scan_scalar_dquot();
6202  if(!_maybe_scan_following_colon())
6203  {
6204  _c4dbgp("seqblck[RVAL]: set as val");
6205  _handle_annotations_before_blck_val_scalar();
6206  csubstr maybe_filtered = _maybe_filter_val_scalar_dquot(sc); // VAL!
6207  m_evt_handler->set_val_scalar_dquoted(maybe_filtered);
6208  addrem_flags(RNXT, RVAL);
6209  }
6210  else
6211  {
6212  _c4dbgp("seqblck[RVAL]: start mapblck, set scalar as key");
6213  addrem_flags(RNXT, RVAL);
6214  _handle_block_check_leading_tabs(startmark);
6215  _handle_annotations_before_start_mapblck(startline);
6216  _handle_colon();
6217  m_evt_handler->begin_map_val_block();
6218  _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
6219  csubstr maybe_filtered = _maybe_filter_key_scalar_dquot(sc); // KEY!
6220  m_evt_handler->set_key_scalar_dquoted(maybe_filtered);
6221  addrem_flags(RMAP|RVAL, RSEQ|RNXT);
6222  _maybe_skip_whitespace_tokens();
6223  goto seqblck_finish;
6224  }
6225  }
6226  // block scalars can only appear as keys when in QMRK scope
6227  // (ie, after ? tokens), so no need to scan following colon in
6228  // here.
6229  else if(first == '|')
6230  {
6231  _c4dbgp("seqblck[RVAL]: block-literal scalar");
6232  ScannedBlock sb;
6233  _scan_block(&sb, m_evt_handler->m_curr->indref + 1);
6234  _handle_annotations_before_blck_val_scalar();
6235  csubstr maybe_filtered = _maybe_filter_val_scalar_literal(sb);
6236  m_evt_handler->set_val_scalar_literal(maybe_filtered);
6237  addrem_flags(RNXT, RVAL);
6238  }
6239  else if(first == '>')
6240  {
6241  _c4dbgp("seqblck[RVAL]: block-folded scalar");
6242  ScannedBlock sb;
6243  _scan_block(&sb, m_evt_handler->m_curr->indref + 1);
6244  _handle_annotations_before_blck_val_scalar();
6245  csubstr maybe_filtered = _maybe_filter_val_scalar_folded(sb);
6246  m_evt_handler->set_val_scalar_folded(maybe_filtered);
6247  addrem_flags(RNXT, RVAL);
6248  }
6249  else if(_scan_scalar_plain_seq_blck(&sc))
6250  {
6251  _c4dbgp("seqblck[RVAL]: plain scalar.");
6252  if(!_maybe_scan_following_colon())
6253  {
6254  _c4dbgp("seqblck[RVAL]: set as val");
6255  _handle_annotations_before_blck_val_scalar();
6256  csubstr maybe_filtered = _maybe_filter_val_scalar_plain(sc, m_evt_handler->m_curr->indref); // VAL!
6257  m_evt_handler->set_val_scalar_plain(maybe_filtered);
6258  addrem_flags(RNXT, RVAL);
6259  }
6260  else
6261  {
6262  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->indref != npos, m_evt_handler->m_curr->pos);
6263  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, startindent > m_evt_handler->m_curr->indref, m_evt_handler->m_curr->pos);
6264  _c4dbgp("seqblck[RVAL]: start mapblck, set scalar as key");
6265  _handle_block_check_leading_tabs(startmark, tabmark);
6266  addrem_flags(RNXT, RVAL);
6267  _handle_annotations_before_start_mapblck(startline);
6268  _handle_colon();
6269  m_evt_handler->begin_map_val_block();
6270  _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
6271  csubstr maybe_filtered = _maybe_filter_key_scalar_plain(sc, m_evt_handler->m_curr->indref); // KEY!
6272  m_evt_handler->set_key_scalar_plain(maybe_filtered);
6273  addrem_flags(RMAP|RVAL, RSEQ|RNXT);
6274  _maybe_skip_whitespace_tokens();
6275  goto seqblck_finish;
6276  }
6277  }
6278  else if(first == '[')
6279  {
6280  _c4dbgp("seqblck[RVAL]: start child seqflow");
6281  addrem_flags(RNXT, RVAL);
6282  _handle_annotations_before_blck_val_scalar();
6283  m_evt_handler->begin_seq_val_flow();
6284  addrem_flags(RFLOW|RVAL, RBLCK|RNXT);
6285  _line_progressed(1);
6286  _set_indentation(m_evt_handler->m_parent->indref + 1u);
6287  goto seqblck_finish;
6288  }
6289  else if(first == '{')
6290  {
6291  _c4dbgp("seqblck[RVAL]: start child mapflow");
6292  addrem_flags(RNXT, RVAL);
6293  _handle_annotations_before_blck_val_scalar();
6294  m_evt_handler->begin_map_val_flow();
6295  addrem_flags(RMAP|RKEY|RFLOW, RBLCK|RSEQ|RVAL|RNXT);
6296  _line_progressed(1);
6297  _set_indentation(m_evt_handler->m_parent->indref + 1u);
6298  goto seqblck_finish;
6299  }
6300  else if(first == '-')
6301  {
6302  _c4dbgp("seqblck[RVAL]: dash");
6303  _handle_block_check_leading_tabs(startmark);
6304  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->indref != npos, m_evt_handler->m_curr->pos);
6305  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, startindent > m_evt_handler->m_curr->indref, m_evt_handler->m_curr->pos);
6306  _c4dbgp("seqblck[RVAL]: start child seqblck");
6307  RYML_ASSERT_PARSE_CB_(this->callbacks(), startindent > m_evt_handler->m_curr->indref, m_evt_handler->m_curr->pos);
6308  addrem_flags(RNXT, RVAL);
6309  _handle_annotations_before_blck_val_scalar();
6310  m_evt_handler->begin_seq_val_block();
6311  addrem_flags(RVAL, RNXT);
6312  _set_indentation(startindent);
6313  // keep going on inside this function
6314  _line_progressed(1);
6315  }
6316  else if(first == ':')
6317  {
6318  _c4dbgp("seqblck[RVAL]: start child mapblck with empty key");
6319  addrem_flags(RNXT, RVAL);
6320  _handle_annotations_before_start_mapblck(startline);
6321  _handle_colon();
6322  m_evt_handler->begin_map_val_block();
6323  _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
6324  m_evt_handler->set_key_scalar_plain_empty();
6325  addrem_flags(RMAP|RVAL, RSEQ|RNXT);
6326  _line_progressed(1);
6327  _maybe_skip_whitespace_tokens();
6328  goto seqblck_finish;
6329  }
6330  else if(first == '&')
6331  {
6332  const csubstr anchor = _scan_anchor();
6333  _c4dbgpf("seqblck[RVAL]: anchor! {}", prs_(anchor));
6334  // we need to buffer the anchors, as there may be two
6335  // consecutive anchors in here
6336  _add_annotation(&m_pending_anchors, anchor, startindent, startline);
6337  }
6338  else if(first == '*')
6339  {
6340  csubstr ref = _scan_ref_seq();
6341  _c4dbgpf("seqblck[RVAL]: ref! {}", prs_(ref));
6342  if(!_maybe_scan_following_colon())
6343  {
6344  _c4dbgp("seqblck[RVAL]: set ref as val!");
6345  _handle_valref(ref);
6346  addrem_flags(RNXT, RVAL);
6347  }
6348  else
6349  {
6350  _c4dbgp("seqblck[RVAL]: ref is key of map");
6351  addrem_flags(RNXT, RVAL);
6352  _handle_annotations_before_start_mapblck(startline);
6353  m_evt_handler->begin_map_val_block();
6354  _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
6355  _handle_keyref(ref);
6356  addrem_flags(RMAP|RVAL, RSEQ|RNXT);
6357  _set_indentation(startindent);
6358  _maybe_skip_whitespace_tokens();
6359  goto seqblck_finish;
6360  }
6361  }
6362  else if(first == '!')
6363  {
6364  csubstr tag = _scan_tag();
6365  _c4dbgpf("seqblck[RVAL]: val tag! {}", prs_(tag));
6366  // we need to buffer the tags, as there may be two
6367  // consecutive tags in here
6368  _add_annotation(&m_pending_tags, tag, startindent, startline);
6369  }
6370  else if(first == '?')
6371  {
6372  _c4dbgp("seqblck[RVAL]: start child mapblck, explicit key");
6373  addrem_flags(RNXT, RVAL);
6374  m_evt_handler->begin_map_val_block();
6375  addrem_flags(RMAP|QMRK, RSEQ|RNXT);
6376  _set_indentation(startindent);
6377  _line_progressed(1);
6378  _maybe_skipchars(' ');
6379  if(_is_blck_seq_token_maybe(m_evt_handler->m_curr->line_contents.rem))
6380  {
6381  _c4dbgp("seqblck[RVAL]: seqblck starts after ?");
6382  addrem_flags(RKCL, QMRK);
6383  m_evt_handler->begin_seq_key_block();
6384  addrem_flags(RSEQ|RVAL, RMAP|RKCL);
6385  _save_indentation();
6386  _line_progressed(1);
6387  _maybe_skipchars(' ');
6388  }
6389  goto seqblck_finish;
6390  }
6391  else
6392  {
6393  _c4err("parse error");
6394  }
6395  }
6396  else // RNXT
6397  {
6398  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RNXT), m_evt_handler->m_curr->pos);
6399  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
6400  //
6401  // handle indentation
6402  //
6403  _c4dbgpf("seqblck[RNXT]: indref={} indentation={}", m_evt_handler->m_curr->indref, m_evt_handler->m_curr->line_contents.indentation);
6404  if C4_LIKELY(m_evt_handler->m_curr->at_line_beginning())
6405  {
6406  _c4dbgp("seqblck[RNXT]: at line begin");
6407  if(m_evt_handler->m_curr->indentation_ge())
6408  {
6409  _c4dbgpf("seqblck[RNXT]: skip {} from indref", m_evt_handler->m_curr->indref);
6410  _line_progressed(m_evt_handler->m_curr->indref);
6411  if(!m_evt_handler->m_curr->line_contents.rem.len)
6412  goto seqblck_again;
6413  }
6414  else if(m_evt_handler->m_curr->indentation_lt())
6415  {
6416  _c4dbgp("seqblck[RNXT]: smaller indentation!");
6417  _handle_indentation_pop_from_block_seq();
6418  if(has_all(RSEQ|RBLCK))
6419  {
6420  _c4dbgp("seqblck[RNXT]: still seqblck!");
6421  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RNXT), m_evt_handler->m_curr->pos);
6422  _line_progressed(m_evt_handler->m_curr->line_contents.indentation);
6423  if(!m_evt_handler->m_curr->line_contents.rem.len)
6424  goto seqblck_again; // LCOV_EXCL_LINE
6425  }
6426  else
6427  {
6428  _c4dbgp("seqblck[RNXT]: no longer seqblck!");
6429  goto seqblck_finish;
6430  }
6431  }
6432  else if(m_evt_handler->m_curr->line_contents.indentation == npos)
6433  {
6434  _c4dbgpf("seqblck[RNXT]: blank line, len={}", m_evt_handler->m_curr->line_contents.rem);
6435  _line_progressed(m_evt_handler->m_curr->line_contents.rem.len);
6436  if(!m_evt_handler->m_curr->line_contents.rem.len)
6437  goto seqblck_again; // LCOV_EXCL_LINE
6438  }
6439  }
6440  else
6441  {
6442  _c4dbgp("seqblck[RNXT]: NOT at line begin");
6443  if(!m_evt_handler->m_curr->line_contents.rem.begins_with_any(" \t"))
6444  {
6445  _c4err("parse error");
6446  }
6447  else
6448  {
6449  _skipchars(" \t");
6450  if(!m_evt_handler->m_curr->line_contents.rem.len)
6451  {
6452  _c4dbgp("seqblck[RNXT]: again");
6453  goto seqblck_again; // LCOV_EXCL_LINE
6454  }
6455  }
6456  }
6457  //
6458  // now handle the tokens
6459  //
6460  _c4assert(m_evt_handler->m_curr->line_contents.rem.len > 0);
6461  const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
6462  _c4dbgpf("seqblck[RNXT]: '{}' node_id={}", _c4prc(first), m_evt_handler->m_curr->node_id);
6463  if(first == '-')
6464  {
6465  if(m_evt_handler->m_curr->indref > 0
6466  || m_evt_handler->m_curr->line_contents.indentation > 0
6467  || !_is_doc_begin_token(m_evt_handler->m_curr->line_contents.rem))
6468  {
6469  if C4_LIKELY(_is_blck_seq_token_maybe(m_evt_handler->m_curr->line_contents.rem))
6470  {
6471  _c4dbgp("seqblck[RNXT]: expect next val");
6472  addrem_flags(RVAL, RNXT);
6473  m_evt_handler->add_sibling();
6474  _line_progressed(1);
6475  }
6476  else
6477  {
6478  _c4err("parse error");
6479  }
6480  }
6481  else
6482  {
6483  _c4dbgp("seqblck[RNXT]: start doc");
6484  _start_doc_suddenly();
6485  _line_progressed(3);
6486  _maybe_skip_whitespace_tokens();
6487  goto seqblck_finish;
6488  }
6489  }
6490  else if(first == ':')
6491  {
6492  // This happens for example in `- [a: b]: c` (after
6493  // terminating the seq, ie, after `]`). All other cases
6494  // (ie colon after scalars) are caught elsewhere (ie, in
6495  // RVAL state).
6496  if C4_LIKELY(m_evt_handler->m_parent && (m_evt_handler->m_parent->flags & RMAP))
6497  {
6498  _c4dbgp("seqblck[RNXT]: actually this seq was '?' key of parent map");
6499  m_evt_handler->end_seq_block();
6500  goto seqblck_finish;
6501  }
6502  else
6503  {
6504  _c4err("parse error");
6505  }
6506  }
6507  else if(first == '.')
6508  {
6509  _c4dbgp("seqblck[RNXT]: maybe doc?");
6510  if(_is_doc_end_token(m_evt_handler->m_curr->line_contents.rem))
6511  {
6512  _c4dbgp("seqblck[RNXT]: end doc");
6513  _end_doc_suddenly();
6514  _line_progressed(3);
6515  _maybe_skip_whitespace_tokens();
6516  _check_doc_end_tokens();
6517  goto seqblck_finish;
6518  }
6519  else
6520  {
6521  _c4err("parse error");
6522  }
6523  }
6524  else
6525  {
6526  // may be an indentless sequence nested in a map...
6527  #ifdef RYML_DBG
6528  _print_state_stack();
6529  #endif
6530  if(m_evt_handler->m_parent
6531  && has_all(RMAP|RBLCK, m_evt_handler->m_parent)
6532  && m_evt_handler->m_curr->indref == m_evt_handler->m_parent->indref)
6533  {
6534  _c4dbgpf("seqblck[RNXT]: end indentless seq, go to parent={}. node={}", m_evt_handler->m_parent->node_id, m_evt_handler->m_curr->node_id);
6535  RYML_ASSERT_PARSE_CB_(this->callbacks(), m_evt_handler->m_curr != m_evt_handler->m_parent, m_evt_handler->m_curr->pos);
6536  _handle_indentation_pop(m_evt_handler->m_parent);
6537  RYML_ASSERT_PARSE_CB_(this->callbacks(), has_all(RMAP|RBLCK), m_evt_handler->m_curr->pos);
6538  m_evt_handler->add_sibling();
6539  addrem_flags(RKEY, RNXT);
6540  goto seqblck_finish;
6541  }
6542  else if(first == '\t')
6543  {
6544  size_t pos = m_evt_handler->m_curr->line_contents.rem.first_not_of('\t');
6545  if(pos == npos)
6546  {
6547  _line_progressed(m_evt_handler->m_curr->line_contents.rem.len);
6548  goto seqblck_again;
6549  }
6550  }
6551  _c4err("parse error");
6552  }
6553  }
6554 
6555  seqblck_again:
6556  _c4dbgt("seqblck: go again", 0);
6557  if(_finished_line())
6558  {
6559  m_bom_len = 0;
6560  _line_ended();
6561  _scan_line();
6562  if(_finished_file())
6563  {
6564  _c4dbgp("seqblck: finish!");
6565  _end_seq_blck();
6566  goto seqblck_finish;
6567  }
6568  _c4dbgnextline();
6569  }
6570  goto seqblck_start;
6571 
6572  seqblck_finish:
6573  _c4dbgp("seqblck: finish");
6574 }
6575 
6576 
6577 //-----------------------------------------------------------------------------
6578 
6579 template<class EventHandler>
6580 void ParseEngine<EventHandler>::_handle_map_block()
6581 {
6582 mapblck_start:
6583  _c4dbgpf("handle_map_block: map_id={} node_id={} level={} indref={}", m_evt_handler->m_parent->node_id, m_evt_handler->m_curr->node_id, m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref);
6584 
6585  // states: RKEY -> RVAL -> RNXT
6586  // states: QMRK -> RKCL -> RVAL -> RNXT
6587  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RMAP), m_evt_handler->m_curr->pos);
6588  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RBLCK), m_evt_handler->m_curr->pos);
6589  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RKEY|RKCL|RVAL|RNXT|QMRK), m_evt_handler->m_curr->pos);
6590  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, 1 == (has_any(RKEY) + has_any(RKCL) + has_any(RVAL) + has_any(RNXT) + has_any(QMRK)), m_evt_handler->m_curr->pos);
6591 
6592  _maybe_skip_comment();
6593  if(!m_evt_handler->m_curr->line_contents.rem.len)
6594  goto mapblck_again;
6595 
6596  if(has_any(RKEY))
6597  {
6598  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
6599  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
6600  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
6601  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
6602  //
6603  // handle indentation
6604  //
6605  if(m_evt_handler->m_curr->at_line_beginning())
6606  {
6607  if(m_evt_handler->m_curr->indentation_eq())
6608  {
6609  _c4dbgpf("mapblck[RKEY]: skip {} from indref", m_evt_handler->m_curr->indref);
6610  _line_progressed(m_evt_handler->m_curr->indref);
6611  if(!m_evt_handler->m_curr->line_contents.rem.len)
6612  goto mapblck_again;
6613  }
6614  else if(m_evt_handler->m_curr->indentation_lt())
6615  {
6616  _c4dbgp("mapblck[RKEY]: smaller indentation!");
6617  _handle_indentation_pop_from_block_map();
6618  _line_progressed(m_evt_handler->m_curr->line_contents.indentation);
6619  if(has_all(RMAP|RBLCK))
6620  {
6621  _c4dbgp("mapblck[RKEY]: still mapblck!");
6622  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RKEY), m_evt_handler->m_curr->pos);
6623  if(!m_evt_handler->m_curr->line_contents.rem.len)
6624  goto mapblck_again;
6625  }
6626  else
6627  {
6628  _c4dbgp("mapblck[RKEY]: no longer mapblck!");
6629  goto mapblck_finish;
6630  }
6631  }
6632  else
6633  {
6634  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->indentation_gt(), m_evt_handler->m_curr->pos);
6635  _c4err("invalid indentation");
6636  }
6637  }
6638  //
6639  // now handle the tokens
6640  //
6641  const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
6642  const size_t startline = m_evt_handler->m_curr->pos.line;
6643  const size_t startindent = m_evt_handler->m_curr->line_contents.current_col();
6644  _c4dbgpf("mapblck[RKEY]: '{}'", _c4prc(first));
6645  ScannedScalar sc;
6646  if(first == '\'')
6647  {
6648  _c4dbgp("mapblck[RKEY]: scanning single-quoted scalar");
6649  sc = _scan_scalar_squot();
6650  csubstr maybe_filtered = _maybe_filter_val_scalar_squot(sc);
6651  _handle_annotations_before_blck_key_scalar();
6652  m_evt_handler->set_key_scalar_squoted(maybe_filtered);
6653  addrem_flags(RVAL, RKEY);
6654  if(!_maybe_scan_following_colon())
6655  _c4err("could not find ':' colon after key");
6656  _handle_colon();
6657  _maybe_skip_whitespace_tokens();
6658  }
6659  else if(first == '"')
6660  {
6661  _c4dbgp("mapblck[RKEY]: scanning double-quoted scalar");
6662  sc = _scan_scalar_dquot();
6663  csubstr maybe_filtered = _maybe_filter_val_scalar_dquot(sc);
6664  _handle_annotations_before_blck_key_scalar();
6665  m_evt_handler->set_key_scalar_dquoted(maybe_filtered);
6666  addrem_flags(RVAL, RKEY);
6667  if(!_maybe_scan_following_colon())
6668  _c4err("could not find ':' colon after key");
6669  _handle_colon();
6670  _maybe_skip_whitespace_tokens();
6671  }
6672  // block scalars (| and >) can not be used as keys unless they
6673  // appear in an explicit QMRK scope (ie, after the ? token),
6674  else if C4_UNLIKELY(first == '|')
6675  {
6676  _c4err("block map: literal keys must be enclosed in '?'");
6677  }
6678  else if C4_UNLIKELY(first == '>')
6679  {
6680  _c4err("block map: folded keys must be enclosed in '?'");
6681  }
6682  else if(_scan_scalar_plain_map_blck(&sc))
6683  {
6684  _c4dbgp("mapblck[RKEY]: plain scalar");
6685  csubstr maybe_filtered = _maybe_filter_val_scalar_plain(sc, m_evt_handler->m_curr->indref);
6686  _handle_annotations_before_blck_key_scalar();
6687  m_evt_handler->set_key_scalar_plain(maybe_filtered);
6688  addrem_flags(RVAL, RKEY);
6689  if(!_maybe_scan_following_colon())
6690  _c4err("could not find ':' colon after key");
6691  _handle_colon();
6692  _maybe_skip_whitespace_tokens();
6693  }
6694  else if(first == '?')
6695  {
6696  _c4dbgp("mapblck[RKEY]: key token!");
6697  addrem_flags(QMRK, RKEY);
6698  _line_progressed(1);
6699  _maybe_skipchars(' ');
6700  if(_is_blck_seq_token_maybe(m_evt_handler->m_curr->line_contents.rem))
6701  {
6702  _c4dbgp("mapblck[RKEY]: seqblck starts after ?");
6703  addrem_flags(RKCL, QMRK);
6704  m_evt_handler->begin_seq_key_block();
6705  addrem_flags(RSEQ|RVAL, RMAP|RKCL);
6706  _save_indentation();
6707  _line_progressed(1);
6708  _maybe_skipchars(' ');
6709  goto mapblck_finish;
6710  }
6711  goto mapblck_again;
6712  }
6713  else if(first == ':')
6714  {
6715  _c4dbgp("mapblck[RKEY]: setting empty key");
6716  _handle_annotations_before_blck_key_scalar();
6717  m_evt_handler->set_key_scalar_plain_empty();
6718  addrem_flags(RVAL, RKEY);
6719  _line_progressed(1);
6720  _handle_colon();
6721  _maybe_skip_whitespace_tokens();
6722  }
6723  else if(first == '*')
6724  {
6725  csubstr ref = _scan_ref_map();
6726  _c4dbgpf("mapblck[RKEY]: key ref! {}", prs_(ref));
6727  _handle_keyref(ref);
6728  addrem_flags(RVAL, RKEY);
6729  if(!_maybe_scan_following_colon())
6730  _c4err("could not find ':' colon after key");
6731  _handle_colon();
6732  _maybe_skip_whitespace_tokens();
6733  }
6734  else if(first == '&')
6735  {
6736  csubstr anchor = _scan_anchor();
6737  _c4dbgpf("mapblck[RKEY]: key anchor! {}", prs_(anchor));
6738  _add_annotation(&m_pending_anchors, anchor, startindent, startline);
6739  }
6740  else if(first == '!')
6741  {
6742  csubstr tag = _scan_tag();
6743  _c4dbgpf("mapblck[RKEY]: key tag! {}", prs_(tag));
6744  _add_annotation(&m_pending_tags, tag, startindent, startline);
6745  }
6746  else if(first == '[')
6747  {
6748  // RYML's tree cannot store container keys, but that's
6749  // handled inside the tree handler. Other handlers may be
6750  // able to handle it.
6751  _c4dbgp("mapblck[RKEY]: start child seqflow (!)");
6752  _handle_annotations_before_blck_key_scalar();
6753  m_evt_handler->begin_seq_key_flow();
6754  addrem_flags(RSEQ|RFLOW|RVAL, RKEY|RMAP|RBLCK);
6755  _line_progressed(1);
6756  _set_indentation(startindent);
6757  goto mapblck_finish;
6758  }
6759  else if(first == '{')
6760  {
6761  // RYML's tree cannot store container keys, but that's
6762  // handled inside the tree handler. Other handlers may be
6763  // able to handle it.
6764  _c4dbgp("mapblck[RKEY]: start child mapflow (!)");
6765  _handle_annotations_before_blck_key_scalar();
6766  m_evt_handler->begin_map_key_flow();
6767  addrem_flags(RFLOW|RKEY, RBLCK);
6768  _line_progressed(1);
6769  _set_indentation(startindent);
6770  goto mapblck_finish;
6771  }
6772  else if(first == '-')
6773  {
6774  _c4dbgp("mapblck[RKEY]: maybe doc?");
6775  if(m_evt_handler->m_curr->line_contents.indentation == 0 && _is_doc_begin_token(m_evt_handler->m_curr->line_contents.rem))
6776  {
6777  _c4dbgp("mapblck[RKEY]: end+start doc");
6778  _start_doc_suddenly();
6779  _line_progressed(3);
6780  _maybe_skip_whitespace_tokens();
6781  goto mapblck_finish;
6782  }
6783  else
6784  {
6785  _c4err("parse error");
6786  }
6787  }
6788  else if(first == '.')
6789  {
6790  _c4dbgp("mapblck[RKEY]: maybe end doc?");
6791  if(m_evt_handler->m_curr->line_contents.indentation == 0 && _is_doc_end_token(m_evt_handler->m_curr->line_contents.rem))
6792  {
6793  _c4dbgp("mapblck[RKEY]: end doc");
6794  _end_doc_suddenly();
6795  _line_progressed(3);
6796  _maybe_skip_whitespace_tokens();
6797  _check_doc_end_tokens();
6798  goto mapblck_finish;
6799  }
6800  else
6801  {
6802  _c4err("parse error"); // LCOV_EXCL_LINE
6803  }
6804  }
6805  else
6806  {
6807  _c4err("parse error");
6808  }
6809  }
6810  else if(has_any(RVAL))
6811  {
6812  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
6813  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
6814  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
6815  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
6816  //
6817  // handle indentation
6818  //
6819  if(m_evt_handler->m_curr->at_line_beginning())
6820  {
6821  _c4dbgpf("mapblck[RVAL]: indref={} indentation={}", m_evt_handler->m_curr->indref+1, m_evt_handler->m_curr->line_contents.indentation);
6822  m_evt_handler->m_curr->more_indented = false;
6823  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->indref != npos, m_evt_handler->m_curr->pos);
6824  if(m_evt_handler->m_curr->indentation_eq_extra())
6825  {
6826  _c4dbgp("mapblck[RVAL]: skip indentation!");
6827  _line_progressed(m_evt_handler->m_curr->indref + 1);
6828  if(!m_evt_handler->m_curr->line_contents.rem.len)
6829  goto mapblck_again;
6830  }
6831  else if(m_evt_handler->m_curr->indentation_gt_extra())
6832  {
6833  _c4dbgp("mapblck[RVAL]: more indented!");
6834  m_evt_handler->m_curr->more_indented = true;
6835  _line_progressed(m_evt_handler->m_curr->line_contents.indentation);
6836  if(!m_evt_handler->m_curr->line_contents.rem.len)
6837  goto mapblck_again; // LCOV_EXCL_LINE
6838  }
6839  else if(m_evt_handler->m_curr->indentation_lt_extra())
6840  {
6841  if(m_evt_handler->m_curr->indentation_eq())
6842  {
6843  _c4dbgp("mapblck[RVAL]: smaller indentation than RVAL!");
6844  // watchout for indentless seqs
6845  if(!_is_blck_seq_token_maybe(m_evt_handler->m_curr->line_contents.rem.sub(m_evt_handler->m_curr->line_contents.indentation)))
6846  {
6847  _c4dbgp("mapblck[RVAL]: smaller indentation than RVAL!");
6848  _handle_annotations_before_blck_val_scalar();
6849  m_evt_handler->set_val_scalar_plain_empty();
6850  addrem_flags(RNXT, RVAL);
6851  goto mapblck_again;
6852  }
6853  }
6854  else
6855  {
6856  _c4dbgp("mapblck[RVAL]: smaller indentation than RKEY!");
6857  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->indentation_lt(), m_evt_handler->m_curr->pos);
6858  _handle_indentation_pop_from_block_map();
6859  if(has_all(RMAP|RBLCK))
6860  {
6861  _c4dbgp("mapblck[RVAL]: still mapblck!");
6862  _line_progressed(m_evt_handler->m_curr->line_contents.indentation);
6863  if(has_any(RNXT))
6864  {
6865  _c4dbgp("mapblck[RVAL]: speculatively expect next keyval");
6866  m_evt_handler->add_sibling();
6867  addrem_flags(RKEY, RNXT);
6868  }
6869  goto mapblck_again;
6870  }
6871  else
6872  {
6873  _c4dbgp("mapblck[RVAL]: no longer mapblck!");
6874  goto mapblck_finish;
6875  }
6876  }
6877  }
6878  }
6879  const size_t startcol = _handle_block_skip_leading_whitespace();
6880  if(startcol == npos)
6881  {
6882  _c4dbgp("mapblck[RVAL]: whitespace only");
6883  goto mapblck_again; // LCOV_EXCL_LINE
6884  }
6885  const size_t tabmark = _handle_block_get_whitespace_mark();
6886  //
6887  // now handle the tokens
6888  //
6889  _c4assert(m_evt_handler->m_curr->line_contents.rem.len);
6890  const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
6891  const size_t startline = m_evt_handler->m_curr->pos.line;
6892  const size_t startindent = m_evt_handler->m_curr->line_contents.current_col();
6893  _c4dbgpf("mapblck[RVAL]: '{}'", _c4prc(first));
6894  ScannedScalar sc;
6895  if(first == '\'')
6896  {
6897  _c4dbgp("mapblck[RVAL]: scanning single-quoted scalar");
6898  sc = _scan_scalar_squot();
6899  if(!_maybe_scan_following_colon())
6900  {
6901  _c4dbgp("mapblck[RVAL]: set as val");
6902  _handle_annotations_before_blck_val_scalar();
6903  csubstr maybe_filtered = _maybe_filter_val_scalar_squot(sc); // VAL!
6904  m_evt_handler->set_val_scalar_squoted(maybe_filtered);
6905  addrem_flags(RNXT, RVAL);
6906  }
6907  else
6908  {
6909  _c4assert(m_evt_handler->m_curr->indref != npos);
6910  _c4assert(startindent > m_evt_handler->m_curr->indref);
6911  _c4dbgp("mapblck[RVAL]: start new block map, set scalar as key");
6912  _handle_block_check_leading_tabs(startcol);
6913  _handle_annotations_before_start_mapblck(startline);
6914  addrem_flags(RNXT, RVAL);
6915  _handle_colon();
6916  m_evt_handler->begin_map_val_block();
6917  _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
6918  csubstr maybe_filtered = _maybe_filter_key_scalar_squot(sc); // KEY!
6919  m_evt_handler->set_key_scalar_squoted(maybe_filtered);
6920  _maybe_skip_whitespace_tokens();
6921  // keep the child state on RVAL
6922  addrem_flags(RVAL, RNXT);
6923  }
6924  }
6925  else if(first == '"')
6926  {
6927  _c4dbgp("mapblck[RVAL]: scanning double-quoted scalar");
6928  sc = _scan_scalar_dquot();
6929  if(!_maybe_scan_following_colon())
6930  {
6931  _c4dbgp("mapblck[RVAL]: set as val");
6932  _handle_annotations_before_blck_val_scalar();
6933  csubstr maybe_filtered = _maybe_filter_val_scalar_dquot(sc); // VAL!
6934  m_evt_handler->set_val_scalar_dquoted(maybe_filtered);
6935  addrem_flags(RNXT, RVAL);
6936  }
6937  else
6938  {
6939  _c4assert(m_evt_handler->m_curr->indref != npos);
6940  _c4assert(startindent > m_evt_handler->m_curr->indref);
6941  _c4dbgp("mapblck[RVAL]: start new block map, set scalar as key");
6942  _handle_block_check_leading_tabs(startcol);
6943  _handle_annotations_before_start_mapblck(startline);
6944  addrem_flags(RNXT, RVAL);
6945  _handle_colon();
6946  m_evt_handler->begin_map_val_block();
6947  _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
6948  csubstr maybe_filtered = _maybe_filter_key_scalar_dquot(sc); // KEY!
6949  m_evt_handler->set_key_scalar_dquoted(maybe_filtered);
6950  _maybe_skip_whitespace_tokens();
6951  // keep the child state on RVAL
6952  addrem_flags(RVAL, RNXT);
6953  }
6954  }
6955  // block scalars can only appear as keys when in QMRK scope
6956  // (ie, after ? tokens), so no need to scan following colon
6957  else if(first == '|')
6958  {
6959  _c4dbgp("mapblck[RVAL]: scanning block-literal scalar");
6960  ScannedBlock sb;
6961  _scan_block(&sb, m_evt_handler->m_curr->indref + 1);
6962  _handle_annotations_before_blck_val_scalar();
6963  csubstr maybe_filtered = _maybe_filter_val_scalar_literal(sb);
6964  m_evt_handler->set_val_scalar_literal(maybe_filtered);
6965  addrem_flags(RNXT, RVAL);
6966  }
6967  else if(first == '>')
6968  {
6969  _c4dbgp("mapblck[RVAL]: scanning block-folded scalar");
6970  ScannedBlock sb;
6971  _scan_block(&sb, m_evt_handler->m_curr->indref + 1);
6972  _handle_annotations_before_blck_val_scalar();
6973  csubstr maybe_filtered = _maybe_filter_val_scalar_folded(sb);
6974  m_evt_handler->set_val_scalar_folded(maybe_filtered);
6975  addrem_flags(RNXT, RVAL);
6976  }
6977  else if(_scan_scalar_plain_map_blck(&sc))
6978  {
6979  _c4dbgp("mapblck[RVAL]: plain scalar.");
6980  if(!_maybe_scan_following_colon())
6981  {
6982  _c4dbgp("mapblck[RVAL]: set as val");
6983  _handle_annotations_before_blck_val_scalar();
6984  csubstr maybe_filtered = _maybe_filter_val_scalar_plain(sc, m_evt_handler->m_curr->indref); // VAL!
6985  m_evt_handler->set_val_scalar_plain(maybe_filtered);
6986  addrem_flags(RNXT, RVAL);
6987  }
6988  else
6989  {
6990  _c4assert(m_evt_handler->m_curr->indref != npos);
6991  _c4assert(startindent > m_evt_handler->m_curr->indref);
6992  _c4dbgpf("mapblck[RVAL]: start new block map, set scalar as key {}", m_evt_handler->m_curr->indref);
6993  _handle_block_check_leading_tabs(startcol, tabmark);
6994  addrem_flags(RNXT, RVAL);
6995  _handle_annotations_before_start_mapblck(startline);
6996  _handle_colon();
6997  m_evt_handler->begin_map_val_block();
6998  _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
6999  csubstr maybe_filtered = _maybe_filter_key_scalar_plain(sc, m_evt_handler->m_curr->indref); // KEY!
7000  m_evt_handler->set_key_scalar_plain(maybe_filtered);
7001  _maybe_skip_whitespace_tokens();
7002  // keep the child state on RVAL
7003  addrem_flags(RVAL, RNXT);
7004  }
7005  }
7006  else if(first == '-' && _is_blck_seq_token_maybe(m_evt_handler->m_curr->line_contents.rem))
7007  {
7008  if C4_UNLIKELY(!m_evt_handler->m_curr->at_first_token())
7009  _c4err("parse error");
7010  _c4dbgp("mapblck[RVAL]: start val seqblck");
7011  _handle_block_check_leading_tabs(startcol);
7012  addrem_flags(RNXT, RVAL);
7013  _handle_annotations_before_blck_val_scalar();
7014  m_evt_handler->begin_seq_val_block();
7015  addrem_flags(RSEQ|RVAL, RMAP|RNXT);
7016  _set_indentation(startindent);
7017  _line_progressed(1);
7018  _maybe_skip_whitespace_tokens();
7019  goto mapblck_finish;
7020  }
7021  else if(first == '[')
7022  {
7023  _c4dbgp("mapblck[RVAL]: start val seqflow");
7024  addrem_flags(RNXT, RVAL);
7025  _handle_annotations_before_blck_val_scalar();
7026  m_evt_handler->begin_seq_val_flow();
7027  addrem_flags(RSEQ|RFLOW|RVAL, RMAP|RBLCK|RNXT);
7028  _set_indentation(m_evt_handler->m_parent->indref + 1u);
7029  _line_progressed(1);
7030  goto mapblck_finish;
7031  }
7032  else if(first == '{')
7033  {
7034  _c4dbgp("mapblck[RVAL]: start val mapflow");
7035  addrem_flags(RNXT, RVAL);
7036  _handle_annotations_before_blck_val_scalar();
7037  m_evt_handler->begin_map_val_flow();
7038  addrem_flags(RKEY|RFLOW, RBLCK|RVAL|RNXT);
7039  m_evt_handler->m_curr->scalar_col = m_evt_handler->m_curr->line_contents.indentation;
7040  _set_indentation(m_evt_handler->m_parent->indref + 1u);
7041  _line_progressed(1);
7042  goto mapblck_finish;
7043  }
7044  else if(first == '*')
7045  {
7046  csubstr ref = _scan_ref_map();
7047  _c4dbgpf("mapblck[RVAL]: ref! {}", prs_(ref));
7048  if(_maybe_scan_following_colon())
7049  {
7050  _c4dbgp("mapblck[RVAL]: start child map, block");
7051  addrem_flags(RNXT, RVAL);
7052  _handle_annotations_before_blck_val_scalar();
7053  m_evt_handler->begin_map_val_block();
7054  _handle_keyref(ref);
7055  _set_indentation(startindent);
7056  // keep going in RVAL
7057  addrem_flags(RVAL, RNXT);
7058  }
7059  else
7060  {
7061  _c4dbgp("mapblck[RVAL]: was val ref");
7062  _handle_valref(ref);
7063  addrem_flags(RNXT, RVAL);
7064  }
7065  _maybe_skip_whitespace_tokens();
7066  }
7067  else if(first == '&')
7068  {
7069  csubstr anchor = _scan_anchor();
7070  _c4dbgpf("mapblck[RVAL]: anchor! {}", prs_(anchor));
7071  // we need to buffer the anchors, as there may be two
7072  // consecutive anchors in here
7073  _add_annotation(&m_pending_anchors, anchor, startindent, startline);
7074  }
7075  else if(first == '!')
7076  {
7077  csubstr tag = _scan_tag();
7078  _c4dbgpf("mapblck[RVAL]: tag! {}", prs_(tag));
7079  // we need to buffer the tags, as there may be two
7080  // consecutive tags in here
7081  _add_annotation(&m_pending_tags, tag, startindent, startline);
7082  }
7083  else if(first == '?')
7084  {
7085  if C4_UNLIKELY(!m_evt_handler->m_curr->at_first_token())
7086  _c4err("parse error");
7087  _c4dbgp("mapblck[RVAL]: start val mapblck");
7088  addrem_flags(RNXT, RVAL);
7089  _handle_annotations_before_blck_val_scalar();
7090  m_evt_handler->begin_map_val_block();
7091  addrem_flags(QMRK, RNXT);
7092  _set_indentation(startindent);
7093  _line_progressed(1);
7094  _maybe_skipchars(' ');
7095  if(_is_blck_seq_token_maybe(m_evt_handler->m_curr->line_contents.rem))
7096  {
7097  _c4dbgp("mapblck[RVAL]: seqblck starts after ?");
7098  addrem_flags(RKCL, QMRK);
7099  m_evt_handler->begin_seq_key_block();
7100  addrem_flags(RSEQ|RVAL, RMAP|RKCL);
7101  _save_indentation();
7102  _line_progressed(1);
7103  _maybe_skipchars(' ');
7104  goto mapblck_finish;
7105  }
7106  goto mapblck_again;
7107  }
7108  else if(first == ':')
7109  {
7110  _c4dbgp("mapblck[RVAL]: start val mapblck");
7111  addrem_flags(RNXT, RVAL);
7112  _handle_annotations_before_start_mapblck(startline);
7113  _handle_colon();
7114  m_evt_handler->begin_map_val_block();
7115  _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
7116  m_evt_handler->set_key_scalar_plain_empty();
7117  // keep the child state on RVAL
7118  addrem_flags(RVAL, RNXT);
7119  _line_progressed(1);
7120  _maybe_skip_whitespace_tokens();
7121  goto mapblck_again;
7122  }
7123  else
7124  {
7125  _c4err("parse error"); // LCOV_EXCL_LINE
7126  }
7127  }
7128  else if(has_any(RNXT))
7129  {
7130  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
7131  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
7132  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
7133  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
7134  //
7135  // handle indentation
7136  //
7137  if(m_evt_handler->m_curr->at_line_beginning())
7138  {
7139  _c4dbgpf("mapblck[RNXT]: indref={} indentation={}", m_evt_handler->m_curr->indref, m_evt_handler->m_curr->line_contents.indentation);
7140  if(m_evt_handler->m_curr->indentation_eq())
7141  {
7142  _c4dbgpf("mapblck[RNXT]: skip {} from indref", m_evt_handler->m_curr->indref);
7143  _line_progressed(m_evt_handler->m_curr->indref);
7144  _c4dbgp("mapblck[RNXT]: speculatively expect next keyval");
7145  m_evt_handler->add_sibling();
7146  addrem_flags(RKEY, RNXT);
7147  goto mapblck_again;
7148  }
7149  else if(m_evt_handler->m_curr->indentation_lt())
7150  {
7151  _c4dbgp("mapblck[RNXT]: smaller indentation!");
7152  _handle_indentation_pop_from_block_map();
7153  if(has_all(RMAP|RBLCK))
7154  {
7155  _line_progressed(m_evt_handler->m_curr->line_contents.indentation);
7156  if(!has_any(RKCL))
7157  {
7158  _c4dbgp("mapblck[RNXT]: speculatively expect next keyval");
7159  m_evt_handler->add_sibling();
7160  addrem_flags(RKEY, RNXT);
7161  }
7162  goto mapblck_again;
7163  }
7164  else
7165  {
7166  goto mapblck_finish;
7167  }
7168  }
7169  }
7170  else
7171  {
7172  _c4dbgp("mapblck[RNXT]: NOT at line begin");
7173  if(!m_evt_handler->m_curr->line_contents.rem.begins_with_any(" \t"))
7174  {
7175  _c4err("parse error");
7176  }
7177  else
7178  {
7179  _skipchars(" \t");
7180  if(!m_evt_handler->m_curr->line_contents.rem.len)
7181  {
7182  _c4dbgp("seqblck[RNXT]: again");
7183  goto mapblck_again; // LCOV_EXCL_LINE
7184  }
7185  }
7186  }
7187  //
7188  // handle tokens
7189  //
7190  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->line_contents.rem.len > 0, m_evt_handler->m_curr->pos);
7191  const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
7192  _c4dbgpf("mapblck[RNXT]: '{}'", _c4prc(first));
7193  if(first == ' ')
7194  {
7195  _c4dbgp("mapblck[RNXT]: skip spaces");
7196  _maybe_skip_whitespace_tokens();
7197  }
7198  else
7199  {
7200  _c4err("parse error");
7201  }
7202  }
7203  else if(has_any(QMRK))
7204  {
7205  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
7206  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
7207  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
7208  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
7209  if(_handle_map_block_qmrk())
7210  goto mapblck_again;
7211  else
7212  goto mapblck_finish;
7213  }
7214  else if(has_any(RKCL)) // read the key colon (after QMRK)
7215  {
7216  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
7217  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
7218  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
7219  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
7220  if(_handle_map_block_rkcl())
7221  goto mapblck_again;
7222  else
7223  goto mapblck_finish;
7224  }
7225 
7226  mapblck_again:
7227  _c4dbgt("mapblck: again", 0);
7228  if(_finished_line())
7229  {
7230  _line_ended();
7231  _scan_line();
7232  if(_finished_file())
7233  {
7234  _c4dbgp("mapblck: file finished!");
7235  _end_map_blck();
7236  goto mapblck_finish;
7237  }
7238  _c4dbgnextline();
7239  }
7240  goto mapblck_start;
7241 
7242  mapblck_finish:
7243  _c4dbgp("mapblck: finish");
7244 }
7245 
7246 
7247 //-----------------------------------------------------------------------------
7248 
7249 // return true if we should remain in map_block
7250 template<class EventHandler>
7251 bool ParseEngine<EventHandler>::_handle_map_block_qmrk()
7252 {
7253  //
7254  // handle indentation
7255  //
7256  if(m_evt_handler->m_curr->at_line_beginning())
7257  {
7258  _c4dbgpf("mapblck[QMRK]: at line beginning. ind={} indref={}", m_evt_handler->m_curr->line_contents.indentation, m_evt_handler->m_curr->indref);
7259  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->line_contents.indentation != npos, m_evt_handler->m_curr->pos);
7260  if(m_evt_handler->m_curr->indentation_eq_extra())
7261  {
7262  _c4dbgpf("mapblck[QMRK]: skip {} from indref", m_evt_handler->m_curr->indref + 1);
7263  _line_progressed(m_evt_handler->m_curr->indref + 1);
7264  if(!m_evt_handler->m_curr->line_contents.rem.len)
7265  return true; // go again
7266  }
7267  // indentation can be larger in QMRK state
7268  else if(m_evt_handler->m_curr->indentation_gt_extra())
7269  {
7270  _c4dbgp("mapblck[QMRK]: larger indentation !");
7271  _line_progressed(m_evt_handler->m_curr->line_contents.indentation);
7272  if(!m_evt_handler->m_curr->line_contents.rem.len)
7273  return true; // go again
7274  }
7275  else
7276  {
7277  _c4dbgp("mapblck[QMRK]: smaller indentation!");
7278  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->indentation_lt_extra(), m_evt_handler->m_curr->pos);
7279  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->line_contents.rem.len > 0, m_evt_handler->m_curr->pos);
7280  if(m_evt_handler->m_curr->indentation_eq()
7281  // defend against docs or indentless seqs
7282  && m_evt_handler->m_curr->line_contents.rem.str[0] != '-')
7283  {
7284  _c4dbgp("mapblck[QMRK]: QMRK finished!");
7285  _handle_annotations_before_blck_key_scalar();
7286  m_evt_handler->set_key_scalar_plain_empty();
7287  addrem_flags(RKCL, QMRK);
7288  return true; // go again
7289  }
7290  else if(m_evt_handler->m_curr->indentation_lt())
7291  {
7292  _c4dbgp("mapblck[QMRK]: indentation pop!");
7293  _handle_indentation_pop_from_block_map();
7294  _line_progressed(m_evt_handler->m_curr->line_contents.indentation);
7295  if(has_all(RMAP|RBLCK))
7296  {
7297  _c4dbgp("mapblck[QMRK]: still mapblck!");
7298  return true; // go again
7299  }
7300  else
7301  {
7302  _c4dbgp("mapblck[QMRK]: no longer mapblck!");
7303  return false; // finish mapblck
7304  }
7305  }
7306  }
7307  }
7308  //
7309  // now handle the tokens
7310  //
7311  _c4assert(m_evt_handler->m_curr->line_contents.rem.len);
7312  const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
7313  const size_t startline = m_evt_handler->m_curr->pos.line;
7314  const size_t startindent = m_evt_handler->m_curr->line_contents.current_col();
7315  _c4dbgpf("mapblck[QMRK]: '{}'", first);
7316  ScannedScalar sc;
7317  if(first == '\'')
7318  {
7319  _c4dbgp("mapblck[QMRK]: scanning single-quoted scalar");
7320  sc = _scan_scalar_squot();
7321  csubstr maybe_filtered = _maybe_filter_key_scalar_squot(sc); // KEY!
7322  addrem_flags(RKCL, QMRK);
7323  if(!_maybe_scan_following_colon())
7324  {
7325  _c4dbgp("mapblck[QMRK]: set as key");
7326  _handle_annotations_before_blck_key_scalar();
7327  m_evt_handler->set_key_scalar_squoted(maybe_filtered);
7328  }
7329  else
7330  {
7331  _c4dbgp("mapblck[QMRK]: start new block map as key (!), set scalar as key");
7332  _handle_annotations_before_start_mapblck_as_key();
7333  m_evt_handler->begin_map_key_block();
7334  _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
7335  m_evt_handler->set_key_scalar_squoted(maybe_filtered);
7336  _maybe_skip_whitespace_tokens();
7337  _set_indentation(startindent);
7338  // keep the child state on RVAL
7339  addrem_flags(RVAL, RKCL);
7340  }
7341  }
7342  else if(first == '"')
7343  {
7344  _c4dbgp("mapblck[QMRK]: scanning double-quoted scalar");
7345  sc = _scan_scalar_dquot();
7346  csubstr maybe_filtered = _maybe_filter_key_scalar_dquot(sc); // KEY!
7347  addrem_flags(RKCL, QMRK);
7348  if(!_maybe_scan_following_colon())
7349  {
7350  _c4dbgp("mapblck[QMRK]: set as key");
7351  _handle_annotations_before_blck_key_scalar();
7352  m_evt_handler->set_key_scalar_dquoted(maybe_filtered);
7353  }
7354  else
7355  {
7356  _c4dbgp("mapblck[QMRK]: start new block map as key (!), set scalar as key");
7357  _handle_annotations_before_start_mapblck_as_key();
7358  m_evt_handler->begin_map_key_block();
7359  _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
7360  m_evt_handler->set_key_scalar_dquoted(maybe_filtered);
7361  _maybe_skip_whitespace_tokens();
7362  _set_indentation(startindent);
7363  // keep the child state on RVAL
7364  addrem_flags(RVAL, RKCL);
7365  }
7366  }
7367  else if(first == '|')
7368  {
7369  _c4dbgp("mapblck[QMRK]: scanning block-literal scalar");
7370  ScannedBlock sb;
7371  _scan_block(&sb, m_evt_handler->m_curr->indref + 1);
7372  csubstr maybe_filtered = _maybe_filter_key_scalar_literal(sb); // KEY!
7373  _handle_annotations_before_blck_key_scalar();
7374  m_evt_handler->set_key_scalar_literal(maybe_filtered);
7375  addrem_flags(RKCL, QMRK);
7376  }
7377  else if(first == '>')
7378  {
7379  _c4dbgp("mapblck[QMRK]: scanning block-literal scalar");
7380  ScannedBlock sb;
7381  _scan_block(&sb, m_evt_handler->m_curr->indref + 1);
7382  csubstr maybe_filtered = _maybe_filter_key_scalar_folded(sb); // KEY!
7383  _handle_annotations_before_blck_key_scalar();
7384  m_evt_handler->set_key_scalar_folded(maybe_filtered);
7385  addrem_flags(RKCL, QMRK);
7386  }
7387  else if(_scan_scalar_plain_map_blck(&sc))
7388  {
7389  _c4dbgp("mapblck[QMRK]: plain scalar");
7390  csubstr maybe_filtered = _maybe_filter_key_scalar_plain(sc, m_evt_handler->m_curr->indref); // KEY!
7391  addrem_flags(RKCL, QMRK);
7392  if(!_maybe_scan_following_colon())
7393  {
7394  _c4dbgp("mapblck[QMRK]: set as key");
7395  _handle_annotations_before_blck_key_scalar();
7396  m_evt_handler->set_key_scalar_plain(maybe_filtered);
7397  }
7398  else
7399  {
7400  _c4dbgp("mapblck[QMRK]: start new block map as key (!), set scalar as key");
7401  _handle_annotations_before_start_mapblck_as_key();
7402  m_evt_handler->begin_map_key_block();
7403  _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
7404  m_evt_handler->set_key_scalar_plain(maybe_filtered);
7405  _maybe_skip_whitespace_tokens();
7406  _set_indentation(startindent);
7407  // keep the child state on RVAL
7408  addrem_flags(RVAL, RKCL);
7409  }
7410  }
7411  else if(first == ':')
7412  {
7413  _c4dbgp("mapblck[QMRK]: start new block map as key (!), empty key");
7414  addrem_flags(RKCL, QMRK);
7415  _handle_annotations_before_start_mapblck_as_key();
7416  m_evt_handler->begin_map_key_block();
7417  _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
7418  m_evt_handler->set_key_scalar_plain_empty();
7419  _line_progressed(1);
7420  _maybe_skip_whitespace_tokens();
7421  _set_indentation(startindent);
7422  // keep the child state on RVAL
7423  addrem_flags(RVAL, RKCL);
7424  }
7425  else if(first == '*')
7426  {
7427  csubstr ref = _scan_ref_map();
7428  _c4dbgpf("mapblck[QMRK]: key ref! {}", prs_(ref));
7429  addrem_flags(RKCL, QMRK);
7430  if(!_maybe_scan_following_colon())
7431  {
7432  _c4dbgp("mapblck[QMRK]: set ref as key");
7433  _handle_keyref(ref);
7434  }
7435  else
7436  {
7437  _c4dbgp("mapblck[QMRK]: start new block map as key (!), set ref as key");
7438  _handle_annotations_before_start_mapblck_as_key();
7439  m_evt_handler->begin_map_key_block();
7440  _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
7441  _handle_keyref(ref);
7442  _set_indentation(startindent);
7443  // keep the child state on RVAL
7444  addrem_flags(RVAL, RKCL|QMRK);
7445  }
7446  _maybe_skip_whitespace_tokens();
7447  }
7448  else if(first == '&')
7449  {
7450  csubstr anchor = _scan_anchor();
7451  _c4dbgpf("mapblck[QMRK]: key anchor! {}", prs_(anchor));
7452  _add_annotation(&m_pending_anchors, anchor, startindent, startline);
7453  }
7454  else if(first == '!')
7455  {
7456  csubstr tag = _scan_tag();
7457  _c4dbgpf("mapblck[QMRK]: key tag! {}", prs_(tag));
7458  _add_annotation(&m_pending_tags, tag, startindent, startline);
7459  }
7460  else if(first == '-')
7461  {
7462  _c4dbgp("mapblck[QMRK]: maybe seq or doc?");
7463  if(_is_blck_seq_token_maybe(m_evt_handler->m_curr->line_contents.rem))
7464  {
7465  _c4dbgp("mapblck[QMRK]: start child seqblck (!)");
7466  addrem_flags(RKCL, QMRK);
7467  _handle_annotations_before_blck_key_scalar();
7468  m_evt_handler->begin_seq_key_block();
7469  addrem_flags(RVAL|RSEQ, RMAP|RKCL);
7470  _set_indentation(startindent);
7471  _line_progressed(1);
7472  }
7473  else
7474  {
7475  _c4dbgp("mapblck[QMRK]: end+start doc");
7476  _c4assert(_is_doc_begin_token(m_evt_handler->m_curr->line_contents.rem));
7477  _start_doc_suddenly();
7478  _line_progressed(3);
7479  }
7480  _maybe_skip_whitespace_tokens();
7481  return false; // finish mapblck
7482  }
7483  else if(first == '[')
7484  {
7485  _c4dbgp("mapblck[QMRK]: start child seqflow (!)");
7486  addrem_flags(RKCL, QMRK);
7487  _handle_annotations_before_blck_key_scalar();
7488  m_evt_handler->begin_seq_key_flow();
7489  addrem_flags(RVAL|RSEQ|RFLOW, RMAP|RKCL|RBLCK);
7490  _set_indentation(m_evt_handler->m_parent->indref + 1);
7491  _line_progressed(1);
7492  return false; // finish mapblck
7493  }
7494  else if(first == '{')
7495  {
7496  _c4dbgp("mapblck[QMRK]: start child mapflow (!)");
7497  addrem_flags(RKCL, QMRK);
7498  _handle_annotations_before_blck_key_scalar();
7499  m_evt_handler->begin_map_key_flow();
7500  addrem_flags(RKEY|RFLOW, RVAL|RKCL|RBLCK);
7501  _set_indentation(m_evt_handler->m_parent->indref + 1);
7502  _line_progressed(1);
7503  return false; // finish mapblck
7504  }
7505  else if(first == '?')
7506  {
7507  _c4dbgpf("mapblck[QMRK]: another QMRK '?'. ind={} indref={}", startindent, m_evt_handler->m_curr->indref);
7508  RYML_ASSERT_PARSE_CB_(callbacks(), startindent > m_evt_handler->m_curr->indref, m_evt_handler->m_curr->pos);
7509  _c4dbgp("mapblck[QMRK]: ? indent gt - start child mapblck (!)");
7510  addrem_flags(RKCL, QMRK);
7511  _handle_annotations_before_blck_key_scalar();
7512  m_evt_handler->begin_map_key_block();
7513  addrem_flags(QMRK, RKCL);
7514  _set_indentation(startindent);
7515  // indentation_lt() should be handled elsewhere
7516  _line_progressed(1);
7517  _maybe_skipchars(' ');
7518  if(_is_blck_seq_token_maybe(m_evt_handler->m_curr->line_contents.rem))
7519  {
7520  _c4dbgp("mapblck[RVAL]: seqblck starts after ?");
7521  addrem_flags(RKCL, QMRK);
7522  m_evt_handler->begin_seq_key_block();
7523  addrem_flags(RSEQ|RVAL, RMAP|RKCL);
7524  _save_indentation();
7525  _line_progressed(1);
7526  _maybe_skipchars(' ');
7527  return false;
7528  }
7529  }
7530  else
7531  {
7532  _c4err("parse error");
7533  }
7534  return true; // continue in mapblck
7535 }
7536 
7537 
7538 //-----------------------------------------------------------------------------
7539 
7540 // return true if we should remain in map_block
7541 template<class EventHandler>
7542 bool ParseEngine<EventHandler>::_handle_map_block_rkcl()
7543 {
7544  //
7545  // handle indentation
7546  //
7547  if(m_evt_handler->m_curr->at_line_beginning())
7548  {
7549  if(m_evt_handler->m_curr->indentation_eq())
7550  {
7551  _c4dbgpf("mapblck[RKCL]: skip {} from indref", m_evt_handler->m_curr->indref);
7552  _line_progressed(m_evt_handler->m_curr->indref);
7553  if(!m_evt_handler->m_curr->line_contents.rem.len)
7554  return true; // continue in mapblck
7555  }
7556  else if C4_UNLIKELY(m_evt_handler->m_curr->indentation_lt())
7557  {
7558  _c4err("invalid indentation");
7559  }
7560  }
7561  const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
7562  _c4dbgpf("mapblck[RKCL]: '{}'", first);
7563  if(first == ':')
7564  {
7565  _c4dbgp("mapblck[RKCL]: found the colon");
7566  _line_progressed(1);
7567  _maybe_skipchars(' ');
7568  #if defined(__GNUC__) && ( \
7569  ((__GNUC__ >= 12) && ((C4_WORDSIZE == 4) || defined(C4_CPU_S390_X) || defined(C4_CPU_PPC64))) \
7570  || \
7571  (__GNUC__ == 16 && defined(C4_CPU_X86_64)))
7572  C4_DONT_OPTIMIZE(m_evt_handler->m_curr->line_contents.rem);
7573  #endif
7574  // sequence is valid after the RKCL ':'
7575  if(!_is_blck_seq_token_maybe(m_evt_handler->m_curr->line_contents.rem))
7576  {
7577  addrem_flags(RVAL, RKCL);
7578  return true; // continue in mapblck
7579  }
7580  else
7581  {
7582  _c4dbgp("mapblck[RKCL]: start val seqblck");
7583  addrem_flags(RNXT, RKCL);
7584  m_evt_handler->begin_seq_val_block();
7585  addrem_flags(RSEQ|RVAL, RMAP|RNXT);
7586  _save_indentation();
7587  _line_progressed(1);
7588  _maybe_skipchars(' ');
7589  return false; // finish mapblck
7590  }
7591  }
7592  else if(first == '?')
7593  {
7594  _c4dbgp("mapblck[RKCL]: got '?'. val was empty");
7595  m_evt_handler->set_val_scalar_plain_empty();
7596  m_evt_handler->add_sibling();
7597  addrem_flags(QMRK, RKCL);
7598  _line_progressed(1);
7599  _maybe_skipchars(' ');
7600  if(_is_blck_seq_token_maybe(m_evt_handler->m_curr->line_contents.rem))
7601  {
7602  _c4dbgp("mapblck[RKCL]: seqblck starts after ?");
7603  addrem_flags(RKCL, QMRK);
7604  m_evt_handler->begin_seq_key_block();
7605  addrem_flags(RSEQ|RVAL, RMAP|QMRK);
7606  _save_indentation();
7607  _line_progressed(1);
7608  _maybe_skipchars(' ');
7609  return false;
7610  }
7611  }
7612  else if(first == '-')
7613  {
7614  if(m_evt_handler->m_curr->indref == 0 || m_evt_handler->m_curr->line_contents.indentation == 0 || _is_doc_begin_token(m_evt_handler->m_curr->line_contents.rem))
7615  {
7616  _c4dbgp("mapblck[RKCL]: end+start doc");
7617  RYML_CHECK_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, _is_doc_begin_token(m_evt_handler->m_curr->line_contents.rem), m_evt_handler->m_curr->pos);
7618  _start_doc_suddenly();
7619  _line_progressed(3);
7620  _maybe_skip_whitespace_tokens();
7621  return false; // finish mapblck
7622  }
7623  else
7624  {
7625  _c4err("parse error"); // LCOV_EXCL_LINE
7626  }
7627  }
7628  else if(first == '.')
7629  {
7630  _c4dbgp("mapblck[RKCL]: maybe end doc?");
7631  csubstr rs = m_evt_handler->m_curr->line_contents.rem.sub(1);
7632  if(rs == ".." || rs.begins_with(".. "))
7633  {
7634  _c4dbgp("mapblck[RKCL]: end+start doc");
7635  _end_doc_suddenly();
7636  _line_progressed(3);
7637  _maybe_skip_whitespace_tokens();
7638  _check_doc_end_tokens();
7639  return false; // finish mapblck
7640  }
7641  else
7642  {
7643  _c4err("parse error"); // LCOV_EXCL_LINE
7644  }
7645  }
7646  else/* if(m_was_inside_qmrk) */
7647  {
7648  _c4dbgp("mapblck[RKCL]: missing :");
7649  if C4_UNLIKELY(!m_evt_handler->m_curr->indentation_eq())
7650  _c4err("parse error"); // LCOV_EXCL_LINE
7651  m_evt_handler->set_val_scalar_plain_empty();
7652  m_evt_handler->add_sibling();
7653  addrem_flags(RKEY, RKCL);
7654  }
7655  return true;
7656 }
7657 
7658 
7659 //-----------------------------------------------------------------------------
7660 
7661 template<class EventHandler>
7662 void ParseEngine<EventHandler>::_handle_unk_json()
7663 {
7664  _c4dbgpf("handle_unk_json indref={} target={}", m_evt_handler->m_curr->indref, m_evt_handler->m_curr->node_id);
7665 
7666  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT|RSEQ|RMAP), m_evt_handler->m_curr->pos);
7667  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RTOP), m_evt_handler->m_curr->pos);
7668 
7669  _maybe_skip_comment();
7670  csubstr rem = m_evt_handler->m_curr->line_contents.rem;
7671  if(!rem.len)
7672  return;
7673 
7674  size_t pos = rem.first_not_of(" \t");
7675  if(pos)
7676  {
7677  pos = pos != npos ? pos : rem.len;
7678  _c4dbgpf("skipping indentation of {}", pos);
7679  _line_progressed(pos);
7680  rem = m_evt_handler->m_curr->line_contents.rem;
7681  if(!rem.len)
7682  return;
7683  _c4dbgpf("rem is now {}", prs_(rem));
7684  }
7685 
7686  if(rem.begins_with('['))
7687  {
7688  _c4dbgp("it's a seq");
7689  _check_trailing_doc_token();
7690  _maybe_begin_doc();
7691  m_evt_handler->begin_seq_val_flow();
7692  addrem_flags(RSEQ|RFLOW|RVAL, RUNK|RTOP|RDOC);
7693  _set_indentation(m_evt_handler->m_curr->line_contents.current_col(rem));
7694  m_doc_empty = false;
7695  _line_progressed(1);
7696  }
7697  else if(rem.begins_with('{'))
7698  {
7699  _c4dbgp("it's a map");
7700  _check_trailing_doc_token();
7701  _maybe_begin_doc();
7702  m_evt_handler->begin_map_val_flow();
7703  addrem_flags(RMAP|RFLOW|RKEY, RVAL|RTOP|RUNK|RDOC);
7704  m_doc_empty = false;
7705  _set_indentation(m_evt_handler->m_curr->line_contents.current_col(rem));
7706  _line_progressed(1);
7707  }
7708  else if(_handle_bom())
7709  {
7710  _c4dbgp("byte order mark");
7711  }
7712  else
7713  {
7714  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, ! has_any(SSCL), m_evt_handler->m_curr->pos);
7715  _maybe_skip_whitespace_tokens();
7716  csubstr s = m_evt_handler->m_curr->line_contents.rem;
7717  if(!s.len)
7718  return;
7719  const size_t startindent = m_evt_handler->m_curr->line_contents.indentation; // save
7720  const char first = s.str[0];
7721  ScannedScalar sc;
7722  if(first == '"')
7723  {
7724  _c4dbgp("runk_json: scanning double-quoted scalar");
7725  _check_trailing_doc_token();
7726  _maybe_begin_doc();
7727  add_flags(RDOC);
7728  m_doc_empty = false;
7729  sc = _scan_scalar_dquot();
7730  csubstr maybe_filtered = _maybe_filter_val_scalar_dquot(sc);
7731  if(!_maybe_scan_following_colon())
7732  {
7733  _c4dbgp("runk_json: set as val");
7734  _handle_annotations_before_blck_val_scalar();
7735  m_evt_handler->set_val_scalar_dquoted(maybe_filtered);
7736  }
7737  else
7738  {
7739  _c4err("parse error");
7740  }
7741  }
7742  else if(_scan_scalar_plain_unk(&sc))
7743  {
7744  _c4dbgp("runk_json: got a plain scalar");
7745  _check_trailing_doc_token();
7746  _maybe_begin_doc();
7747  add_flags(RDOC);
7748  m_doc_empty = false;
7749  if(!_maybe_scan_following_colon())
7750  {
7751  _c4dbgp("runk_json: set as val");
7752  _handle_annotations_before_blck_val_scalar();
7753  csubstr maybe_filtered = _maybe_filter_val_scalar_plain(sc, startindent);
7754  m_evt_handler->set_val_scalar_plain(maybe_filtered);
7755  }
7756  else
7757  {
7758  _c4err("parse error"); // LCOV_EXCL_LINE
7759  }
7760  }
7761  else
7762  {
7763  _c4err("parse error"); // LCOV_EXCL_LINE
7764  }
7765  }
7766 }
7767 
7768 
7769 //-----------------------------------------------------------------------------
7770 
7771 template<class EventHandler>
7772 void ParseEngine<EventHandler>::_handle_unk()
7773 {
7774  _c4dbgpf("handle_unk indref={} target={}", m_evt_handler->m_curr->indref, m_evt_handler->m_curr->node_id);
7775 
7776  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT|RSEQ|RMAP), m_evt_handler->m_curr->pos);
7777  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RTOP), m_evt_handler->m_curr->pos);
7778 
7779  _maybe_skipchars(' ');
7780  _maybe_skip_comment();
7781 
7782  if(!m_evt_handler->m_curr->line_contents.rem.len)
7783  return;
7784 
7785  _c4dbgpf("runk: rem is now {}", prs_(m_evt_handler->m_curr->line_contents.rem));
7786 
7787  if(m_evt_handler->m_curr->line_contents.indentation == 0u && (m_evt_handler->m_curr->at_line_beginning() || (m_bom_len && (m_evt_handler->m_curr->pos.line == m_bom_line))))
7788  {
7789  _c4dbgpf("runk: rtop: zero indent + at line begin. offset={}", m_evt_handler->m_curr->pos.offset);
7790  _c4dbgp("runk: check BOM");
7791  if(_handle_bom())
7792  {
7793  m_bom_line = m_evt_handler->m_curr->pos.line;
7794  _c4dbgpf("runk: byte order mark! line={} offset={}", m_bom_line, m_evt_handler->m_curr->pos.offset);
7795  return;
7796  }
7797  const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
7798  _c4dbgpf("runk: rtop: first={}", _c4prc(first));
7799  if(first == '-')
7800  {
7801  _c4dbgp("runk: rtop: suspecting doc");
7802  if(_is_doc_begin_token(m_evt_handler->m_curr->line_contents.rem))
7803  {
7804  _c4dbgp("runk: rtop: begin doc");
7805  _maybe_end_doc();
7806  _begin2_doc_expl();
7807  _set_indentation(0);
7808  addrem_flags(RDOC|RUNK, NDOC);
7809  _line_progressed(3u);
7810  _maybe_skip_whitespace_tokens();
7811  return;
7812  }
7813  }
7814  else if(first == '.')
7815  {
7816  _c4dbgp("runk: rtop: suspecting doc end");
7817  if(_is_doc_end_token(m_evt_handler->m_curr->line_contents.rem))
7818  {
7819  _c4dbgp("runk: rtop: end doc");
7820  if(has_any(RDOC))
7821  {
7822  _end2_doc_expl();
7823  }
7824  else
7825  {
7826  _c4dbgp("runk: rtop: ignore end doc");
7827  }
7828  addrem_flags(NDOC|RUNK, RDOC);
7829  _line_progressed(3u);
7830  _maybe_skip_whitespace_tokens();
7831  _check_doc_end_tokens();
7832  return;
7833  }
7834  }
7835  else if(first == '%')
7836  {
7837  _c4dbgpf("directive: {}", m_evt_handler->m_curr->line_contents.rem);
7838  if C4_UNLIKELY(has_any(RDOC) || (!m_doc_empty && has_none(NDOC)))
7839  _c4err("need document footer before directives");
7840  _handle_directive(m_evt_handler->m_curr->line_contents.rem);
7841  return;
7842  }
7843  }
7844 
7845  /* no else-if! */
7846 
7847  size_t startindent = m_evt_handler->m_curr->line_contents.indentation;
7848  size_t remindent = m_evt_handler->m_curr->line_contents.current_col(m_evt_handler->m_curr->line_contents.rem);
7849  if(m_bom_len)
7850  {
7851  _c4dbgpf("runk: prev BOMlen={}", m_bom_len);
7852  if(m_evt_handler->m_curr->pos.line == m_bom_line)
7853  {
7854  _c4dbgpf("runk: BOM remindent={} offset={}", remindent, m_evt_handler->m_curr->pos.offset);
7855  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, remindent >= m_bom_len, m_evt_handler->m_curr->pos);
7856  remindent -= m_bom_len;
7857  }
7858  else
7859  {
7860  m_bom_len = 0;
7861  }
7862  }
7863 
7864  size_t startcol = _handle_block_skip_leading_whitespace();
7865  const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
7866 
7867  if(first == '[')
7868  {
7869  _c4dbgp("runk: flow seq?");
7870  _handle_unk_begin_doc();
7871  if C4_LIKELY( ! _annotations_require_key_container())
7872  {
7873  _c4dbgp("runk: it's a seq, flow");
7874  _handle_annotations_before_blck_val_scalar();
7875  m_evt_handler->begin_seq_val_flow();
7876  addrem_flags(RSEQ|RFLOW|RVAL, RUNK|RTOP|RDOC);
7877  _set_indentation(0);
7878  }
7879  else
7880  {
7881  _c4dbgp("runk: start new block map, set flow seq as key (!)");
7882  _handle_annotations_before_start_mapblck(m_evt_handler->m_curr->pos.line);
7883  m_evt_handler->begin_map_val_block();
7884  addrem_flags(RMAP|RBLCK|RKEY, RUNK|RTOP|RDOC);
7885  _handle_annotations_and_indentation_after_start_mapblck(remindent, m_evt_handler->m_curr->pos.line);
7886  m_evt_handler->begin_seq_key_flow();
7887  addrem_flags(RSEQ|RFLOW|RVAL, RMAP|RBLCK|RKEY);
7888  _set_indentation(0);
7889  }
7890  _line_progressed(1);
7891  }
7892  else if(first == '{')
7893  {
7894  _c4dbgp("runk: flow map?");
7895  _handle_unk_begin_doc();
7896  if C4_LIKELY( ! _annotations_require_key_container())
7897  {
7898  _c4dbgp("runk: it's a map, flow");
7899  _handle_annotations_before_blck_val_scalar();
7900  m_evt_handler->begin_map_val_flow();
7901  addrem_flags(RMAP|RFLOW|RKEY, RVAL|RTOP|RUNK|RDOC);
7902  _set_indentation(0);
7903  }
7904  else
7905  {
7906  _c4dbgp("runk: start new block map, set flow map as key (!)");
7907  _handle_annotations_before_start_mapblck(m_evt_handler->m_curr->pos.line);
7908  m_evt_handler->begin_map_val_block();
7909  addrem_flags(RMAP|RBLCK|RKEY, RUNK|RTOP|RDOC);
7910  _handle_annotations_and_indentation_after_start_mapblck(remindent, m_evt_handler->m_curr->pos.line);
7911  m_evt_handler->begin_map_key_flow();
7912  addrem_flags(RMAP|RFLOW, RBLCK);
7913  _set_indentation(0);
7914  }
7915  _line_progressed(1);
7916  }
7917  else if(first == '-' && _is_blck_token(m_evt_handler->m_curr->line_contents.rem))
7918  {
7919  _c4dbgp("runk: it's a seq, block");
7920  if C4_UNLIKELY(!m_evt_handler->m_curr->at_first_token())
7921  startindent = _handle_unk_check_left_tokens(startindent, m_evt_handler->m_curr->pos.col, /*skip_annotations*/false);
7922  _handle_unk_begin_doc();
7923  _handle_annotations_before_blck_val_scalar();
7924  m_evt_handler->begin_seq_val_block();
7925  addrem_flags(RSEQ|RBLCK|RVAL, RNXT|RTOP|RUNK|RDOC);
7926  _set_indentation(startindent);
7927  _line_progressed(1);
7928  _maybe_skipchars(' ');
7929  }
7930  else if(first == '?' && _is_blck_token(m_evt_handler->m_curr->line_contents.rem))
7931  {
7932  _c4dbgp("runk: it's a map + this key is complex");
7933  if C4_UNLIKELY(!m_evt_handler->m_curr->at_first_token())
7934  startindent = _handle_unk_check_left_tokens(startindent, m_evt_handler->m_curr->pos.col, /*skip_annotations*/false);
7935  _handle_block_check_leading_tabs(startcol);
7936  _handle_unk_begin_doc();
7937  _handle_annotations_before_blck_val_scalar();
7938  m_evt_handler->begin_map_val_block();
7939  addrem_flags(RMAP|RBLCK|QMRK, RKEY|RVAL|RTOP|RUNK|RDOC);
7940  _set_indentation(startindent);
7941  _line_progressed(1);
7942  _maybe_skipchars(' ');
7943  if(_is_blck_seq_token_maybe(m_evt_handler->m_curr->line_contents.rem))
7944  {
7945  _c4dbgp("runk: seqblck key starts after ?");
7946  addrem_flags(RKCL, QMRK);
7947  m_evt_handler->begin_seq_key_block();
7948  addrem_flags(RSEQ|RVAL, RMAP|RKCL);
7949  _save_indentation();
7950  _line_progressed(1);
7951  _maybe_skipchars(' ');
7952  }
7953  }
7954  else if(first == ':' && _is_blck_token(m_evt_handler->m_curr->line_contents.rem))
7955  {
7956  if(m_doc_empty || (m_pending_anchors.num_entries | m_pending_tags.num_entries))
7957  {
7958  _c4dbgp("runk: it's a map with an empty key");
7959  if C4_UNLIKELY(!m_evt_handler->m_curr->at_first_token())
7960  startindent = _handle_unk_check_left_tokens(startindent, m_evt_handler->m_curr->pos.col);
7961  _handle_block_check_leading_tabs(startcol);
7962  const size_t startline = m_evt_handler->m_curr->pos.line; // save
7963  _handle_unk_begin_doc();
7964  _handle_annotations_before_start_mapblck(startline);
7965  _handle_colon();
7966  m_evt_handler->begin_map_val_block();
7967  _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
7968  m_evt_handler->set_key_scalar_plain_empty();
7969  _set_indentation(startindent);
7970  }
7971  else
7972  {
7973  _c4err("block colon cannot occur on a new line unless ? is used");
7974  }
7975  addrem_flags(RMAP|RBLCK|RVAL, RTOP|RUNK|RDOC);
7976  _line_progressed(1);
7977  _maybe_skip_whitespace_tokens();
7978  }
7979  else if(first == '&')
7980  {
7981  csubstr anchor = _scan_anchor();
7982  _c4dbgpf("anchor! {}", prs_(anchor));
7983  const size_t line = m_evt_handler->m_curr->pos.line;
7984  _handle_unk_begin_doc();
7985  _add_annotation(&m_pending_anchors, anchor, remindent, line);
7986  _set_indentation(0);
7987  }
7988  else if(first == '*')
7989  {
7990  csubstr ref = _scan_ref_map();
7991  _c4dbgpf("runk: ref! {}", prs_(ref));
7992  _handle_unk_begin_doc();
7993  if(!_maybe_scan_following_colon())
7994  {
7995  _c4dbgp("runk: set val ref");
7996  _handle_valref(ref);
7997  }
7998  else
7999  {
8000  _c4dbgp("runk: start new block map, set ref as key");
8001  _handle_block_check_leading_tabs(startcol);
8002  const size_t startline = m_evt_handler->m_curr->pos.line; // save
8003  _handle_annotations_before_start_mapblck(startline);
8004  m_evt_handler->begin_map_val_block();
8005  _handle_keyref(ref);
8006  _maybe_skip_whitespace_tokens();
8007  _set_indentation(0);
8008  addrem_flags(RMAP|RBLCK|RVAL, RTOP|RUNK|RDOC);
8009  }
8010  }
8011  else if(first == '!')
8012  {
8013  csubstr tag_orig;
8014  csubstr tag = _scan_tag(&tag_orig);
8015  _c4dbgpf("runk: val tag! {}", prs_(tag));
8016  // we need to buffer the tags, as there may be two
8017  // consecutive tags in here
8018  const size_t indentation = m_evt_handler->m_curr->line_contents.current_col(m_evt_handler->m_curr->line_contents.rem);
8019  const size_t line = m_evt_handler->m_curr->pos.line;
8020  _add_annotation(&m_pending_tags, tag, indentation, line, tag_orig);
8021  }
8022  else
8023  {
8024  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, ! has_any(SSCL), m_evt_handler->m_curr->pos);
8025  const size_t startscalar = _handle_block_get_whitespace_mark();
8026  const size_t startline = m_evt_handler->m_curr->pos.line; // save
8027  auto beginmap = [&](size_t startindent_){
8028  if C4_UNLIKELY(m_evt_handler->m_curr->pos.line > startline)
8029  _c4err("multiline scalars cannot be used as implicit keys");
8030  _handle_block_check_leading_tabs(startcol, startscalar);
8031  _handle_annotations_before_start_mapblck(startline);
8032  _handle_colon();
8033  m_evt_handler->begin_map_val_block();
8034  _handle_annotations_and_indentation_after_start_mapblck(startindent_, startline);
8035  };
8036  auto after_beginmap = [&](size_t startindent_){
8037  _maybe_skip_whitespace_tokens();
8038  _set_indentation(startindent_);
8039  addrem_flags(RMAP|RBLCK|RVAL, RTOP|RUNK|RDOC);
8040  };
8041  if(first == '|')
8042  {
8043  _c4dbgp("runk: block-literal scalar");
8044  _handle_unk_begin_doc();
8045  ScannedBlock sb;
8046  _scan_block(&sb, startindent);
8047  _handle_annotations_before_blck_val_scalar();
8048  csubstr maybe_filtered = _maybe_filter_val_scalar_literal(sb);
8049  m_evt_handler->set_val_scalar_literal(maybe_filtered);
8050  }
8051  else if(first == '>')
8052  {
8053  _c4dbgp("runk: block-folded scalar");
8054  _handle_unk_begin_doc();
8055  ScannedBlock sb;
8056  _scan_block(&sb, startindent);
8057  _handle_annotations_before_blck_val_scalar();
8058  csubstr maybe_filtered = _maybe_filter_val_scalar_folded(sb);
8059  m_evt_handler->set_val_scalar_folded(maybe_filtered);
8060  }
8061  else if(first == '\'')
8062  {
8063  _c4dbgp("runk: single-quoted scalar");
8064  _handle_unk_begin_doc();
8065  bool firsttoken = m_evt_handler->m_curr->at_first_token();
8066  size_t col = m_evt_handler->m_curr->pos.col;
8067  ScannedScalar sc = _scan_scalar_squot();
8068  if(!_maybe_scan_following_colon())
8069  {
8070  _c4dbgp("runk: set as val");
8071  _handle_annotations_before_blck_val_scalar();
8072  csubstr maybe_filtered = _maybe_filter_val_scalar_squot(sc);
8073  m_evt_handler->set_val_scalar_squoted(maybe_filtered);
8074  }
8075  else
8076  {
8077  _c4dbgp("runk: start new block map, set single-quoted scalar as key");
8078  if C4_UNLIKELY(m_evt_handler->m_curr->pos.line > startline)
8079  _c4err("multiline key");
8080  if(!firsttoken)
8081  startindent = _handle_unk_check_left_tokens(startindent, col);
8082  beginmap(startindent);
8083  csubstr maybe_filtered = _maybe_filter_val_scalar_squot(sc);
8084  m_evt_handler->set_key_scalar_squoted(maybe_filtered);
8085  after_beginmap(startindent);
8086  }
8087  }
8088  else if(first == '"')
8089  {
8090  _c4dbgp("runk: double-quoted scalar");
8091  _handle_unk_begin_doc();
8092  bool firsttoken = m_evt_handler->m_curr->at_first_token();
8093  size_t col = m_evt_handler->m_curr->pos.col;
8094  ScannedScalar sc = _scan_scalar_dquot();
8095  if(!_maybe_scan_following_colon())
8096  {
8097  _c4dbgp("runk: set as val");
8098  _handle_annotations_before_blck_val_scalar();
8099  csubstr maybe_filtered = _maybe_filter_val_scalar_dquot(sc);
8100  m_evt_handler->set_val_scalar_dquoted(maybe_filtered);
8101  }
8102  else
8103  {
8104  _c4dbgp("runk: start new block map, set double-quoted scalar as key");
8105  if C4_UNLIKELY(m_evt_handler->m_curr->pos.line > startline)
8106  _c4err("multiline key");
8107  if(!firsttoken)
8108  startindent = _handle_unk_check_left_tokens(startindent, col);
8109  beginmap(startindent);
8110  csubstr maybe_filtered = _maybe_filter_val_scalar_dquot(sc);
8111  m_evt_handler->set_key_scalar_dquoted(maybe_filtered);
8112  after_beginmap(startindent);
8113  }
8114  }
8115  else
8116  {
8117  bool firsttoken = m_evt_handler->m_curr->at_first_token();
8118  size_t col = m_evt_handler->m_curr->pos.col;
8119  ScannedScalar sc;
8120  if(_scan_scalar_plain_unk(&sc))
8121  {
8122  _c4dbgp("runk: plain scalar");
8123  _handle_unk_begin_doc();
8124  if(!_maybe_scan_following_colon())
8125  {
8126  _c4dbgp("runk: set as val");
8127  _handle_annotations_before_blck_val_scalar();
8128  csubstr maybe_filtered = _maybe_filter_val_scalar_plain(sc, startindent);
8129  m_evt_handler->set_val_scalar_plain(maybe_filtered);
8130  }
8131  else
8132  {
8133  _c4dbgp("runk: start new block map, set plain scalar as key");
8134  // there is already a check to multiline inside
8135  // _scan_scalar_plain_unk(), so we don't need to
8136  // throw an error here. but let's be safe by
8137  // asserting the assumption:
8138  _c4assert(m_evt_handler->m_curr->pos.line == startline);
8139  if(!firsttoken)
8140  startindent = _handle_unk_check_left_tokens(startindent, col);
8141  beginmap(startindent);
8142  csubstr maybe_filtered = _maybe_filter_val_scalar_plain(sc, startindent);
8143  m_evt_handler->set_key_scalar_plain(maybe_filtered);
8144  after_beginmap(startindent);
8145  }
8146  }
8147  else
8148  {
8149  _c4err("parse error"); // LCOV_EXCL_LINE
8150  }
8151  }
8152  }
8153 
8154  if(m_bom_len && has_none(RUNK))
8155  {
8156  _c4dbgpf("runk: BOMlen={} BOMline={} now={} at_end={}", m_bom_len, m_bom_line, m_evt_handler->m_curr->pos.line, !m_evt_handler->m_curr->line_contents.rem.len);
8157  if(m_evt_handler->m_curr->pos.line != m_bom_line || !m_evt_handler->m_curr->line_contents.rem.len)
8158  {
8159  _c4dbgp("runk: clear BOMlen");
8160  m_bom_len = 0;
8161  }
8162  }
8163 }
8164 
8165 template<class EventHandler>
8166 void ParseEngine<EventHandler>::_handle_unk_begin_doc()
8167 {
8168  _c4dbgp("runk: begin doc");
8169  _check_trailing_doc_token();
8170  _maybe_begin_doc();
8171  add_flags(RDOC);
8172  m_doc_empty = false;
8173 }
8174 
8175 template<class EventHandler>
8176 size_t ParseEngine<EventHandler>::_handle_unk_check_left_tokens(size_t realindent, size_t col, bool skip_annotations)
8177 {
8178  _c4assert(col >= 1);
8179  col -= 1;
8180  _c4assert(col >= m_bom_len);
8181  csubstr s = m_evt_handler->m_curr->line_contents.full.range(m_bom_len, col);
8182  size_t pos = 0;
8183  _c4dbgpf("runk: check left tokens: s={}", prs_(s, /*escape*/true));
8184  if(skip_annotations)
8185  {
8186  _handle_unk_get_first_non_pending_token_pos(s, &realindent, &pos);
8187  _c4dbgpf("runk: skip annotations: realindent={} pos={}", realindent, pos);
8188  }
8189  size_t firstns = s.first_not_of(' ', pos);
8190  if(firstns == npos)
8191  firstns = s.len;
8192  _c4dbgpf("runk: check left tokens:\n"
8193  " tokens={} skipped={}\n"
8194  " bomlen={} first={} col={}\n"
8195  " (bomlen+first)={} vs {}=col\n"
8196  " startindent={} lineindent={}"
8197  , prs_(s, /*escape*/true), prs_(s.sub(firstns), /*escape*/true)
8198  , m_bom_len, firstns, col
8199  , m_bom_len+firstns, col,
8200  realindent, m_evt_handler->m_curr->line_contents.indentation);
8201  if(m_bom_len + firstns != col)
8202  _c4err("parse error");
8203  if(!skip_annotations)
8204  realindent = firstns;
8205  _c4dbgpf("runk: pos={} firstns={} -> realindent={}", pos, firstns, realindent);
8206  return realindent;
8207 }
8208 
8209 
8210 /** skip annotations which are pending on the same line */
8211 template<class EventHandler>
8212 void ParseEngine<EventHandler>::_handle_unk_get_first_non_pending_token_pos(csubstr s, size_t *indent, size_t *first_non_token_pos)
8213 {
8214  csubstr first, second;
8215  uint32_t total = _get_annotations_same_line(s, &first, &second);
8216  _c4dbgpf("runk: before skip: {}", prs_(s, true));
8217  size_t pos = s.first_not_of(" \t");
8218  if(pos == npos)
8219  pos = s.len;
8220  if(!total)
8221  {
8222  *indent = *first_non_token_pos = pos;
8223  return;
8224  }
8225  _c4assert(!s.sub(pos).begins_with_any(" \t"));
8226  _c4dbgpf("runk: after skip leading {} whitespace: {}", pos, prs_(s.sub(pos), true));
8227  _c4dbgpf("runk: first annotation: {}", first);
8228  _c4assert(first.len);
8229  _c4assert(first.is_sub(s));
8230  _c4assert(first.is_sub(s.sub(pos)));
8231  _c4assert(s.sub(pos).begins_with(first));
8232  *indent = pos;
8233  pos += first.len;
8234  _c4dbgpf("runk: after skip first annotation: pos={} {}", pos, prs_(s.sub(pos), true));
8235  if(total > 1)
8236  {
8237  _c4dbgpf("runk: second annotation: {}", second);
8238  _c4assert(total == 2);
8239  _c4assert(second.len);
8240  _c4assert(second.is_sub(s));
8241  _c4assert(second.is_sub(s.sub(pos)));
8242  csubstr spos = s.sub(pos);
8243  size_t more = spos.first_not_of(" \t");
8244  _c4assert(more != npos); // because the annotations are on the same line
8245  _c4dbgpf("runk: next nonspace: {}", pos + more);
8246  pos += more;
8247  _c4dbgpf("runk: after skip annotation whitespace: pos={} {}", pos, prs_(s.sub(pos), true));
8248  _c4assert(s.sub(pos).begins_with(second));
8249  pos += second.len;
8250  _c4dbgpf("runk: after skip annotation 2: pos={} {}", pos, prs_(s.sub(pos), true));
8251  }
8252  *first_non_token_pos = pos;
8253 }
8254 
8255 
8256 template<class EventHandler>
8257 uint32_t ParseEngine<EventHandler>::_get_annotations_same_line(csubstr token_soup, csubstr *first_, csubstr *second_) const
8258 {
8259  _c4assert(!m_evt_handler->m_curr->at_first_token());
8260  (void)token_soup;
8261  using EntryPtr = typename Annotation::Entry const* C4_RESTRICT;
8262  EntryPtr first = nullptr;
8263  EntryPtr second = nullptr;
8264  uint32_t total = (uint32_t)(m_pending_anchors.num_entries + m_pending_tags.num_entries);
8265  if(total)
8266  {
8267  _c4dbgpf("there are {} pending annotations: {} anchors + {} tags", total, m_pending_anchors.num_entries, m_pending_tags.num_entries);
8268  auto valid_if_same_line = [this](EntryPtr entry){
8269  _c4dbgpf("pending: {} indent={} line={} vs currline={}", maybe_null_str_(entry->str), entry->indentation, entry->line, m_evt_handler->m_curr->pos.line);
8270  return (entry->line == m_evt_handler->m_curr->pos.line) ? entry : nullptr;
8271  };
8272  // now select annotations only on the same line
8273  total = 0;
8274  for(size_t i = 0; i < m_pending_anchors.num_entries; ++i)
8275  total += !!valid_if_same_line(&m_pending_anchors.annotations[i]);
8276  for(size_t i = 0; i < m_pending_tags.num_entries; ++i)
8277  total += !!valid_if_same_line(&m_pending_tags.annotations[i]);
8278  _c4dbgpf("{} annotations on same line", total);
8279  _c4assert(total > 0); // because this function is only called
8280  // while not at the first token. That
8281  // means we must have same-line
8282  // annotations.
8283  auto get_first_on_same_line = [this](EntryPtr not_this_one){
8284  for(size_t i = 0; i < m_pending_anchors.num_entries; ++i)
8285  if(&m_pending_anchors.annotations[i] != not_this_one
8286  && m_pending_anchors.annotations[i].line == m_evt_handler->m_curr->pos.line)
8287  return &m_pending_anchors.annotations[i];
8288  for(size_t i = 0; i < m_pending_tags.num_entries; ++i)
8289  if(&m_pending_tags.annotations[i] != not_this_one
8290  && m_pending_tags.annotations[i].line == m_evt_handler->m_curr->pos.line)
8291  return &m_pending_tags.annotations[i];
8292  C4_UNREACHABLE(); // LCOV_EXCL_LINE
8293  return (EntryPtr)nullptr; // LCOV_EXCL_LINE
8294  };
8295  _c4assert(total >= 1);
8296  // assign to first
8297  first = get_first_on_same_line(nullptr);
8298  _c4assert(first);
8299  _c4dbgpf("first annotation: {} indent={} line={}", maybe_null_str_(first->str), first->indentation, first->line);
8300  if(total > 1)
8301  {
8302  _c4assert(total == 2);
8303  // assign to second
8304  second = get_first_on_same_line(first);
8305  _c4assert(second);
8306  _c4dbgpf("second annotation: {} indent={} line={}", maybe_null_str_(second->str), second->indentation, second->line);
8307  }
8308  auto extract_string = [&](EntryPtr e){
8309  // tags can be null when the arena ran out of space
8310  if(!e->str.str || e->str.begins_with_any("!<"))
8311  {
8312  csubstr tag = e->orig;
8313  _c4assert(tag.str);
8314  _c4assert(tag.len);
8315  _c4assert(tag.is_sub(token_soup));
8316  _c4dbgpf("tag: {} -> {}", maybe_null_str_(e->str), tag);
8317  return tag;
8318  }
8319  csubstr anchor = e->str;
8320  _c4assert(anchor.len);
8321  _c4assert(anchor.str);
8322  _c4assert(anchor.is_sub(token_soup));
8323  _c4assert(!anchor.begins_with('&'));
8324  _c4assert(anchor.str - token_soup.str > 0);
8325  // add back the anchor's &
8326  --anchor.str;
8327  ++anchor.len;
8328  _c4assert(anchor.begins_with('&'));
8329  _c4dbgpf("anchor: {} -> {}", e->str, anchor);
8330  return anchor;
8331  };
8332  *first_ = first ? extract_string(first) : nullptr;
8333  *second_ = second ? extract_string(second) : nullptr;
8334  if(total > 1 && (first_->str > second_->str))
8335  {
8336  csubstr tmp = *first_;
8337  *first_ = *second_;
8338  *second_ = tmp;
8339  _c4dbgpf("swap first and second: {} -> {}", *first_, *second_);
8340  }
8341  }
8342  return total;
8343 }
8344 
8345 
8346 //-----------------------------------------------------------------------------
8347 
8348 template<class EventHandler>
8349 C4_COLD void ParseEngine<EventHandler>::_handle_usty()
8350 {
8351  _c4dbgpf("handle_usty target={}", m_evt_handler->m_curr->indref, m_evt_handler->m_curr->node_id);
8352 
8353  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RBLCK|RFLOW), m_evt_handler->m_curr->pos);
8354 
8355  #ifdef RYML_NO_COVERAGE__TO_BE_DELETED
8356  if(has_any(RNXT))
8357  {
8358  _c4dbgp("usty[RNXT]: finishing!");
8359  _end_stream();
8360  }
8361  #endif
8362 
8363  _maybe_skip_comment();
8364  csubstr rem = m_evt_handler->m_curr->line_contents.rem;
8365  if(!rem.len)
8366  return;
8367 
8368  size_t pos = rem.first_not_of(" \t");
8369  if(pos)
8370  {
8371  pos = pos != npos ? pos : rem.len;
8372  _c4dbgpf("skipping indentation of {}", pos);
8373  _line_progressed(pos);
8374  rem = m_evt_handler->m_curr->line_contents.rem;
8375  if(!rem.len)
8376  return;
8377  _c4dbgpf("rem is now {}", prs_(rem));
8378  }
8379 
8380  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, rem.len > 0, m_evt_handler->m_curr->pos);
8381  size_t startindent = m_evt_handler->m_curr->line_contents.indentation; // save
8382  char first = rem.str[0];
8383  if(has_any(RSEQ)) // destination is a sequence
8384  {
8385  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, ! has_any(RMAP), m_evt_handler->m_curr->pos);
8386  _c4dbgpf("usty[RSEQ]: first='{}'", _c4prc(first));
8387  if(first == '[')
8388  {
8389  _c4dbgp("usty[RSEQ]: it's a flow seq. merging it");
8390  add_flags(RNXT);
8391  m_evt_handler->_push();
8392  addrem_flags(RFLOW|RVAL, RNXT|USTY);
8393  _set_indentation(startindent);
8394  _line_progressed(1);
8395  _maybe_skip_whitespace_tokens();
8396  }
8397  else if(first == '-' && _is_blck_token(rem))
8398  {
8399  _c4dbgp("usty[RSEQ]: it's a block seq. merging it");
8400  add_flags(RNXT);
8401  m_evt_handler->_push();
8402  addrem_flags(RBLCK|RVAL, RNXT|USTY);
8403  _set_indentation(startindent);
8404  _line_progressed(1);
8405  _maybe_skip_whitespace_tokens();
8406  }
8407  else
8408  {
8409  _c4err("can only parse a seq into an existing seq");
8410  }
8411  }
8412  else if(has_any(RMAP)) // destination is a map
8413  {
8414  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, ! has_any(RSEQ), m_evt_handler->m_curr->pos);
8415  _c4dbgpf("usty[RMAP]: first='{}'", _c4prc(first));
8416  if(first == '{')
8417  {
8418  _c4dbgp("usty[RMAP]: it's a flow map. merging it");
8419  add_flags(RNXT);
8420  _handle_annotations_before_blck_val_scalar();
8421  m_evt_handler->_push();
8422  addrem_flags(RMAP|RFLOW|RKEY, RNXT|USTY);
8423  _set_indentation(startindent);
8424  _line_progressed(1);
8425  _maybe_skip_whitespace_tokens();
8426  }
8427  else if(first == '?' && _is_blck_token(rem))
8428  {
8429  _c4dbgp("usty[RMAP]: it's a block map + this key is complex");
8430  add_flags(RNXT);
8431  _handle_annotations_before_blck_val_scalar();
8432  m_evt_handler->_push();
8433  addrem_flags(RMAP|RBLCK|QMRK, RNXT|USTY);
8434  _save_indentation();
8435  _line_progressed(1);
8436  _maybe_skip_whitespace_tokens();
8437  }
8438  else if(first == ':' && _is_blck_token(rem))
8439  {
8440  _c4dbgp("usty[RMAP]: it's a map with an empty key");
8441  add_flags(RNXT);
8442  _handle_annotations_before_blck_val_scalar();
8443  m_evt_handler->_push();
8444  m_evt_handler->set_key_scalar_plain_empty();
8445  addrem_flags(RMAP|RBLCK|RVAL, RNXT|USTY);
8446  _save_indentation();
8447  _line_progressed(1);
8448  _maybe_skip_whitespace_tokens();
8449  }
8450  else if(rem.begins_with('&'))
8451  {
8452  csubstr anchor = _scan_anchor();
8453  _c4dbgpf("usty[RMAP]: anchor! {}", prs_(anchor));
8454  const size_t indentation = m_evt_handler->m_curr->line_contents.current_col(rem);
8455  const size_t line = m_evt_handler->m_curr->pos.line;
8456  _add_annotation(&m_pending_anchors, anchor, indentation, line);
8457  _set_indentation(m_evt_handler->m_curr->line_contents.current_col(rem));
8458  }
8459  else if(first == '*')
8460  {
8461  csubstr ref = _scan_ref_map();
8462  _c4dbgpf("usty[RMAP]: ref! {}", prs_(ref));
8463  if(!_maybe_scan_following_colon())
8464  {
8465  _c4err("cannot read a VAL to a map");
8466  }
8467  else
8468  {
8469  _c4dbgp("usty[RMAP]: start new block map, set ref as key");
8470  const size_t startline = m_evt_handler->m_curr->pos.line; // save
8471  add_flags(RNXT);
8472  _handle_annotations_before_start_mapblck(startline);
8473  m_evt_handler->_push();
8474  _handle_keyref(ref);
8475  _maybe_skip_whitespace_tokens();
8476  _set_indentation(startindent);
8477  addrem_flags(RMAP|RBLCK|RVAL, RNXT|USTY);
8478  }
8479  }
8480  else if(first == '!')
8481  {
8482  csubstr tag = _scan_tag();
8483  _c4dbgpf("usty[RMAP]: val tag! {}", prs_(tag));
8484  // we need to buffer the tags, as there may be two
8485  // consecutive tags in here
8486  const size_t indentation = m_evt_handler->m_curr->line_contents.current_col(rem);
8487  const size_t line = m_evt_handler->m_curr->pos.line;
8488  _add_annotation(&m_pending_tags, tag, indentation, line);
8489  }
8490  else if(first == '[' || (first == '-' && _is_blck_token(rem)))
8491  {
8492  _c4err("cannot parse a seq into an existing map");
8493  }
8494  else
8495  {
8496  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, ! has_any(SSCL), m_evt_handler->m_curr->pos);
8497  startindent = m_evt_handler->m_curr->line_contents.indentation; // save
8498  const size_t startline = m_evt_handler->m_curr->pos.line; // save
8499  ScannedScalar sc;
8500  _c4dbgpf("usty[RMAP]: maybe scalar. first='{}'", _c4prc(first));
8501  if(first == '\'')
8502  {
8503  _c4dbgp("usty[RMAP]: scanning single-quoted scalar");
8504  sc = _scan_scalar_squot();
8505  if(!_maybe_scan_following_colon())
8506  {
8507  _c4err("cannot read a VAL to a map");
8508  }
8509  else
8510  {
8511  _c4dbgp("usty[RMAP]: start new block map, set scalar as key");
8512  add_flags(RNXT);
8513  _handle_annotations_before_start_mapblck(startline);
8514  m_evt_handler->_push();
8515  _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
8516  csubstr maybe_filtered = _maybe_filter_key_scalar_squot(sc);
8517  m_evt_handler->set_key_scalar_squoted(maybe_filtered);
8518  _set_indentation(startindent);
8519  addrem_flags(RMAP|RBLCK|RVAL, RNXT|USTY);
8520  _maybe_skip_whitespace_tokens();
8521  }
8522  }
8523  else if(first == '"')
8524  {
8525  _c4dbgp("usty[RMAP]: scanning double-quoted scalar");
8526  sc = _scan_scalar_dquot();
8527  if(!_maybe_scan_following_colon())
8528  {
8529  _c4err("cannot read a VAL to a map");
8530  }
8531  else
8532  {
8533  _c4dbgp("usty[RMAP]: start new block map, set double-quoted scalar as key");
8534  add_flags(RNXT);
8535  _handle_annotations_before_start_mapblck(startline);
8536  m_evt_handler->_push();
8537  _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
8538  csubstr maybe_filtered = _maybe_filter_key_scalar_dquot(sc);
8539  m_evt_handler->set_key_scalar_dquoted(maybe_filtered);
8540  _set_indentation(startindent);
8541  addrem_flags(RMAP|RBLCK|RVAL, RNXT|USTY);
8542  _maybe_skip_whitespace_tokens();
8543  }
8544  }
8545  else if(first == '|')
8546  {
8547  _c4err("block literal keys must be enclosed in '?'");
8548  }
8549  else if(first == '>')
8550  {
8551  _c4err("block literal keys must be enclosed in '?'");
8552  }
8553  else if(_scan_scalar_plain_unk(&sc))
8554  {
8555  _c4dbgp("usty[RMAP]: got a plain scalar");
8556  if(!_maybe_scan_following_colon())
8557  {
8558  _c4err("cannot read a VAL to a map");
8559  }
8560  else
8561  {
8562  _c4dbgp("usty[RMAP]: start new block map, set scalar as key");
8563  add_flags(RNXT);
8564  _handle_annotations_before_start_mapblck(startline);
8565  m_evt_handler->_push();
8566  _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
8567  csubstr maybe_filtered = _maybe_filter_key_scalar_plain(sc, startindent);
8568  m_evt_handler->set_key_scalar_plain(maybe_filtered);
8569  _set_indentation(startindent);
8570  addrem_flags(RMAP|RBLCK|RVAL, RNXT|USTY);
8571  _maybe_skip_whitespace_tokens();
8572  }
8573  }
8574  else
8575  {
8576  _c4err("parse error"); // LCOV_EXCL_LINE
8577  }
8578  }
8579  }
8580  else // destination is unknown
8581  {
8582  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, ! has_any(RSEQ), m_evt_handler->m_curr->pos);
8583  _c4dbgpf("usty[UNK]: first='{}'", _c4prc(first));
8584  if(first == '[')
8585  {
8586  _c4dbgp("usty[UNK]: it's a flow seq");
8587  add_flags(RNXT);
8588  _handle_annotations_before_blck_val_scalar();
8589  m_evt_handler->begin_seq_val_flow();
8590  addrem_flags(RSEQ|RFLOW|RVAL, RNXT|USTY);
8591  _set_indentation(startindent);
8592  _line_progressed(1);
8593  _maybe_skip_whitespace_tokens();
8594  }
8595  else if(first == '-' && _is_blck_token(rem))
8596  {
8597  _c4dbgp("usty[UNK]: it's a block seq");
8598  add_flags(RNXT);
8599  _handle_annotations_before_blck_val_scalar();
8600  m_evt_handler->begin_seq_val_block();
8601  addrem_flags(RSEQ|RBLCK|RVAL, RNXT|USTY);
8602  _set_indentation(startindent);
8603  _line_progressed(1);
8604  _maybe_skip_whitespace_tokens();
8605  }
8606  else if(first == '{')
8607  {
8608  _c4dbgp("usty[UNK]: it's a flow map");
8609  add_flags(RNXT);
8610  _handle_annotations_before_blck_val_scalar();
8611  m_evt_handler->begin_map_val_flow();
8612  addrem_flags(RMAP|RFLOW|RKEY, RNXT|USTY);
8613  _set_indentation(startindent);
8614  _line_progressed(1);
8615  _maybe_skip_whitespace_tokens();
8616  }
8617  else if(first == '?' && _is_blck_token(rem))
8618  {
8619  _c4dbgp("usty[UNK]: it's a map + this key is complex");
8620  add_flags(RNXT);
8621  _handle_annotations_before_blck_val_scalar();
8622  m_evt_handler->begin_map_val_block();
8623  addrem_flags(RMAP|RBLCK|QMRK, RNXT|USTY);
8624  _save_indentation();
8625  _line_progressed(1);
8626  _maybe_skip_whitespace_tokens();
8627  }
8628  else if(first == ':' && _is_blck_token(rem))
8629  {
8630  _c4dbgp("usty[UNK]: it's a map with an empty key");
8631  add_flags(RNXT);
8632  _handle_annotations_before_blck_val_scalar();
8633  m_evt_handler->begin_map_val_block();
8634  m_evt_handler->set_key_scalar_plain_empty();
8635  addrem_flags(RMAP|RBLCK|RVAL, RNXT|USTY);
8636  _save_indentation();
8637  _line_progressed(1);
8638  _maybe_skip_whitespace_tokens();
8639  }
8640  else if(first == '&')
8641  {
8642  csubstr anchor = _scan_anchor();
8643  _c4dbgpf("usty[UNK]: anchor! {}", prs_(anchor));
8644  const size_t indentation = m_evt_handler->m_curr->line_contents.current_col(rem);
8645  const size_t line = m_evt_handler->m_curr->pos.line;
8646  _add_annotation(&m_pending_anchors, anchor, indentation, line);
8647  _set_indentation(m_evt_handler->m_curr->line_contents.current_col(rem));
8648  }
8649  else if(first == '*')
8650  {
8651  csubstr ref = _scan_ref_map();
8652  _c4dbgpf("usty[UNK]: ref! {}", prs_(ref));
8653  if(!_maybe_scan_following_colon())
8654  {
8655  _c4dbgp("usty[UNK]: set val ref");
8656  _handle_valref(ref);
8657  }
8658  else
8659  {
8660  _c4dbgp("usty[UNK]: start new block map, set ref as key");
8661  const size_t startline = m_evt_handler->m_curr->pos.line; // save
8662  add_flags(RNXT);
8663  _handle_annotations_before_start_mapblck(startline);
8664  m_evt_handler->begin_map_val_block();
8665  _handle_keyref(ref);
8666  _maybe_skip_whitespace_tokens();
8667  _set_indentation(startindent);
8668  addrem_flags(RMAP|RBLCK|RVAL, RNXT|USTY);
8669  }
8670  }
8671  else if(first == '!')
8672  {
8673  csubstr tag = _scan_tag();
8674  _c4dbgpf("usty[UNK]: val tag! {}", prs_(tag));
8675  // we need to buffer the tags, as there may be two
8676  // consecutive tags in here
8677  const size_t indentation = m_evt_handler->m_curr->line_contents.current_col(rem);
8678  const size_t line = m_evt_handler->m_curr->pos.line;
8679  _add_annotation(&m_pending_tags, tag, indentation, line);
8680  }
8681  else
8682  {
8683  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, ! has_any(SSCL), m_evt_handler->m_curr->pos);
8684  startindent = m_evt_handler->m_curr->line_contents.indentation; // save
8685  const size_t startline = m_evt_handler->m_curr->pos.line; // save
8686  first = rem.str[0];
8687  ScannedScalar sc;
8688  _c4dbgpf("usty[UNK]: maybe scalar. first='{}'", _c4prc(first));
8689  if(first == '\'')
8690  {
8691  _c4dbgp("usty[UNK]: scanning single-quoted scalar");
8692  sc = _scan_scalar_squot();
8693  if(!_maybe_scan_following_colon())
8694  {
8695  _c4dbgp("usty[UNK]: set as val");
8696  _handle_annotations_before_blck_val_scalar();
8697  csubstr maybe_filtered = _maybe_filter_val_scalar_squot(sc);
8698  m_evt_handler->set_val_scalar_squoted(maybe_filtered);
8699  _end_stream();
8700  }
8701  else
8702  {
8703  _c4dbgp("usty[UNK]: start new block map, set scalar as key");
8704  add_flags(RNXT);
8705  _handle_annotations_before_start_mapblck(startline);
8706  m_evt_handler->begin_map_val_block();
8707  _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
8708  csubstr maybe_filtered = _maybe_filter_key_scalar_squot(sc);
8709  m_evt_handler->set_key_scalar_squoted(maybe_filtered);
8710  _set_indentation(startindent);
8711  addrem_flags(RMAP|RBLCK|RVAL, RNXT|USTY);
8712  _maybe_skip_whitespace_tokens();
8713  }
8714  }
8715  else if(first == '"')
8716  {
8717  _c4dbgp("usty[UNK]: scanning double-quoted scalar");
8718  sc = _scan_scalar_dquot();
8719  if(!_maybe_scan_following_colon())
8720  {
8721  _c4dbgp("usty[UNK]: set as val");
8722  _handle_annotations_before_blck_val_scalar();
8723  csubstr maybe_filtered = _maybe_filter_val_scalar_dquot(sc);
8724  m_evt_handler->set_val_scalar_dquoted(maybe_filtered);
8725  _end_stream();
8726  }
8727  else
8728  {
8729  _c4dbgp("usty[UNK]: start new block map, set double-quoted scalar as key");
8730  add_flags(RNXT);
8731  _handle_annotations_before_start_mapblck(startline);
8732  m_evt_handler->begin_map_val_block();
8733  _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
8734  csubstr maybe_filtered = _maybe_filter_key_scalar_dquot(sc);
8735  m_evt_handler->set_key_scalar_dquoted(maybe_filtered);
8736  _set_indentation(startindent);
8737  addrem_flags(RMAP|RBLCK|RVAL, RNXT|USTY);
8738  _maybe_skip_whitespace_tokens();
8739  }
8740  }
8741  else if(first == '|')
8742  {
8743  _c4dbgp("usty[UNK]: scanning block-literal scalar");
8744  ScannedBlock sb;
8745  _scan_block(&sb, startindent);
8746  _c4dbgp("usty[UNK]: set as val");
8747  _handle_annotations_before_blck_val_scalar();
8748  csubstr maybe_filtered = _maybe_filter_val_scalar_literal(sb);
8749  m_evt_handler->set_val_scalar_literal(maybe_filtered);
8750  _end_stream();
8751  }
8752  else if(first == '>')
8753  {
8754  _c4dbgp("usty[UNK]: scanning block-folded scalar");
8755  ScannedBlock sb;
8756  _scan_block(&sb, startindent);
8757  _c4dbgp("usty[UNK]: set as val");
8758  _handle_annotations_before_blck_val_scalar();
8759  csubstr maybe_filtered = _maybe_filter_val_scalar_folded(sb);
8760  m_evt_handler->set_val_scalar_folded(maybe_filtered);
8761  _end_stream();
8762  }
8763  else if(_scan_scalar_plain_unk(&sc))
8764  {
8765  _c4dbgp("usty[UNK]: got a plain scalar");
8766  if(!_maybe_scan_following_colon())
8767  {
8768  _c4dbgp("usty[UNK]: set as val");
8769  _handle_annotations_before_blck_val_scalar();
8770  csubstr maybe_filtered = _maybe_filter_val_scalar_plain(sc, startindent);
8771  m_evt_handler->set_val_scalar_plain(maybe_filtered);
8772  _end_stream();
8773  }
8774  else
8775  {
8776  _c4dbgp("usty[UNK]: start new block map, set scalar as key");
8777  add_flags(RNXT);
8778  _handle_annotations_before_start_mapblck(startline);
8779  m_evt_handler->begin_map_val_block();
8780  _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
8781  csubstr maybe_filtered = _maybe_filter_key_scalar_plain(sc, startindent);
8782  m_evt_handler->set_key_scalar_plain(maybe_filtered);
8783  _set_indentation(startindent);
8784  addrem_flags(RMAP|RBLCK|RVAL, RNXT|USTY);
8785  _maybe_skip_whitespace_tokens();
8786  }
8787  }
8788  else
8789  {
8790  _c4err("parse error"); // LCOV_EXCL_LINE
8791  }
8792  }
8793  }
8794 }
8795 
8796 
8797 //-----------------------------------------------------------------------------
8798 
8799 template<class EventHandler>
8800 void ParseEngine<EventHandler>::parse_json_in_place_ev(csubstr filename, substr src)
8801 {
8802  RYML_ASSERT_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_stack.size() >= 1);
8803  RYML_SAVE_TEST_JSON_(filename, src);
8804  m_evt_handler->start_parse(filename.str, src);
8805  m_evt_handler->begin_stream();
8806  _reset();
8807  while( ! _finished_file())
8808  {
8809  _scan_line();
8810  while( ! _finished_line())
8811  {
8812  _c4dbgnextline();
8813  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, ! m_evt_handler->m_curr->line_contents.rem.empty(), m_evt_handler->m_curr->pos);
8814  if(has_any(RSEQ))
8815  {
8816  _handle_seq_json();
8817  }
8818  else if(has_any(RMAP))
8819  {
8820  _handle_map_json();
8821  }
8822  else if(has_any(RUNK))
8823  {
8824  _handle_unk_json();
8825  }
8826  else
8827  {
8828  _c4err("internal error"); // LCOV_EXCL_LINE
8829  }
8830  }
8831  if(_finished_file())
8832  break; // it may have finished because of multiline blocks
8833  _line_ended();
8834  }
8835  _end_stream();
8836  m_evt_handler->finish_parse();
8837 }
8838 
8839 
8840 //-----------------------------------------------------------------------------
8841 
8842 template<class EventHandler>
8843 void ParseEngine<EventHandler>::parse_in_place_ev(csubstr filename, substr src)
8844 {
8845  RYML_ASSERT_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_stack.size() >= 1);
8846  RYML_SAVE_TEST_YAML_(filename, src);
8847  m_evt_handler->start_parse(filename.str, src);
8848  m_evt_handler->begin_stream();
8849  _reset();
8850  while( ! _finished_file())
8851  {
8852  _scan_line();
8853  while( ! _finished_line())
8854  {
8855  _c4dbgnextline();
8856  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, ! m_evt_handler->m_curr->line_contents.rem.empty(), m_evt_handler->m_curr->pos);
8857  if(has_any(RFLOW))
8858  {
8859  if(has_none(RSEQIMAP))
8860  {
8861  if(has_any(RSEQ))
8862  {
8863  _handle_seq_flow();
8864  }
8865  else
8866  {
8867  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RMAP), m_evt_handler->m_curr->pos);
8868  _handle_map_flow();
8869  }
8870  }
8871  else
8872  {
8873  _handle_seq_imap();
8874  }
8875  }
8876  else if(has_any(RBLCK))
8877  {
8878  if(has_any(RSEQ))
8879  {
8880  _handle_seq_block();
8881  }
8882  else
8883  {
8884  RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RMAP), m_evt_handler->m_curr->pos);
8885  _handle_map_block();
8886  }
8887  }
8888  else if(has_any(RUNK))
8889  {
8890  _handle_unk();
8891  }
8892  else if(has_any(USTY))
8893  {
8894  _handle_usty();
8895  }
8896  else
8897  {
8898  _c4err("internal error"); // LCOV_EXCL_LINE
8899  }
8900  }
8901  if(_finished_file())
8902  break; // it may have finished because of multiline blocks
8903  _line_ended();
8904  }
8905  _end_stream();
8906  m_evt_handler->finish_parse();
8907 }
8908 /** @endcond */
8909 
8910 } // namespace yml
8911 } // namespace c4
8912 
8913 // NOLINTEND(hicpp-signed-bitwise,cppcoreguidelines-avoid-goto,hicpp-avoid-goto,hicpp-multiway-paths-covered,modernize-avoid-c-style-cast)
8914 
8915 #undef _c4dbgnextline
8916 #undef _c4assert
8917 #undef _c4err
8918 
8919 C4_SUPPRESS_WARNING_MSVC_POP
8920 C4_SUPPRESS_WARNING_GCC_CLANG_POP
8921 
8922 #endif // C4_YML_PARSE_ENGINE_DEF_HPP_
c4::yml::Encoding_e
enum c4::yml::Encoding_ Encoding_e
c4::yml::to_chars
size_t to_chars(substr buf, escaped_scalar e)
formatting implementation to escape a scalar with escape_scalar()
Definition: escape_scalar.hpp:264
c4::yml::is_valid_tag_handle
bool is_valid_tag_handle(csubstr handle)
Definition: tag.cpp:210
_ryml_relocate
#define _ryml_relocate(s)
c4::yml::NONE
an index to none
Definition: common.hpp:131
c4::yml::ParseEngine::filter_scalar_block_folded
FilterResult filter_scalar_block_folded(csubstr scalar, substr dst, size_t indentation, BlockChomp_e chomp)
filter a block-folded scalar
c4::yml::RUNK
reading unknown state (when starting): must determine whether scalar, map or seq
Definition: parser_state.hpp:19
c4::yml::ParseEngine::val_location
Location val_location(const char *val) const
Given a pointer to a buffer position, get the location.
c4::yml::RBLCK
reading in block mode
Definition: parser_state.hpp:23
c4::yml::ParseEngine::filter_scalar_block_folded_in_place
FilterResult filter_scalar_block_folded_in_place(substr scalar, size_t cap, size_t indentation, BlockChomp_e chomp)
filter a block-folded scalar in place
_c4assert
#define _c4assert(...)
Definition: parse_engine.def.hpp:37
c4::yml::USTY
reading in unknown style mode - must determine FLOW or BLCK reading an implicit map nested in an expl...
Definition: parser_state.hpp:34
c4::yml::RKEY
reading a key
Definition: parser_state.hpp:25
c4::yml::Location::name
csubstr name
name of the file
Definition: common.hpp:233
c4::basic_substring::str
C * str
a restricted pointer to the first character of the substring
Definition: substr.hpp:216
c4::basic_substring::first_of
size_t first_of(const C c, size_t start=0) const
Definition: substr.hpp:934
node_type.hpp
flags
EventBits flags
Definition: ints_utils.cpp:27
str
const char * str
Definition: node_type.cpp:70
c4::fmt::hex
integral_< intptr_t > hex(std::nullptr_t)
format null as an hexadecimal value
Definition: format.hpp:160
c4::yml::RDOC
reading a document
Definition: parser_state.hpp:32
c4::yml::ParseEngine::parse_in_place_ev
void parse_in_place_ev(csubstr filename, substr src)
parse YAML in place, emitting events to the current handler
c4::yml::RFLOW
reading is inside explicit flow chars: [] or {}
Definition: parser_state.hpp:22
c4::yml::err_parse
void err_parse(ErrorDataParse const &errdata, const char *msg)
trigger a parse error to its respective handler, with a non-formatted error message.
Definition: common.cpp:210
c4::basic_substring::first
basic_substring first(size_t num) const noexcept
return the first num elements: [0,num[
Definition: substr.hpp:529
c4::yml::SSCL
there's a stored scalar
Definition: parser_state.hpp:29
c4::yml::ParseEngine::parse_json_in_place_ev
void parse_json_in_place_ev(csubstr filename, substr src)
parse JSON in place, emitting events to the current handler
c4::yml::RVAL
reading a val
Definition: parser_state.hpp:26
c4::yml::id_type
RYML_ID_TYPE id_type
The type of a node id in the YAML tree; to override the default type, define the macro RYML_ID_TYPE t...
Definition: common.hpp:124
c4::yml::RNXT
read next sibling
Definition: parser_state.hpp:28
c4::yml::ParserOptions
Options to give to the ParseEngine to control its behavior.
Definition: parse_options.hpp:21
c4::yml::UTF8
UTF8.
Definition: common.hpp:144
c4::yml::adjust_pos_with_escapes
size_t adjust_pos_with_escapes(csubstr scalar, size_t pos, bool keep_newlines=false)
Adjust a position in a scalar, increasing it to account for any escaped characters.
Definition: escape_scalar.hpp:172
RYML_LOCATIONS_SMALL_THRESHOLD
#define RYML_LOCATIONS_SMALL_THRESHOLD
threshold at which a location search will revert from linear to binary search.
Definition: common.hpp:39
c4::basic_substring::sub
basic_substring sub(size_t first) const noexcept
return [first,len[
Definition: substr.hpp:502
charconv.hpp
Lightweight generic type-safe wrappers for converting individual values to/from strings.
RYML_NOEXCEPT
#define RYML_NOEXCEPT
Conditionally expands to noexcept when RYML_USE_ASSERT is 0 and is empty otherwise.
Definition: doxy_common.hpp:290
c4::yml::RSEQ
reading a seq
Definition: parser_state.hpp:21
c4::substr
basic_substring< char > substr
a mutable string view
Definition: substr.hpp:2355
c4::yml::npos
a null string position
Definition: common.hpp:138
c4::csubstr
basic_substring< const char > csubstr
an immutable string view
Definition: substr.hpp:2356
RYML_WITH_OR_WITHOUT_TAB_TOKENS_
#define RYML_WITH_OR_WITHOUT_TAB_TOKENS_(with, without)
Definition: parse_engine.def.hpp:48
c4::yml::RKCL
reading the key colon (ie the : after the key in the map)
Definition: parser_state.hpp:27
c4::yml::NOBOM
No Byte Order Mark was found.
Definition: common.hpp:143
c4::atou
bool atou(csubstr str, T *v) noexcept
Convert a trimmed string to an unsigned integral value.
Definition: charconv.hpp:1558
c4::yml::QMRK
reading an explicit key (? key)
Definition: parser_state.hpp:24
c4::yml::RMAP
reading a map
Definition: parser_state.hpp:20
c4::basic_substring::range
basic_substring range(size_t first, size_t last=npos) const noexcept
return [first,last[.
Definition: substr.hpp:519
c4::yml::RTOP
reading at top level
Definition: parser_state.hpp:18
c4::basic_substring::triml
basic_substring triml(const C c) const
trim left
Definition: substr.hpp:629
c4::yml::ParseEngine
This is the main driver of parsing logic: it scans the YAML or JSON source for tokens, and emits the appropriate sequence of parsing events to its event handler.
Definition: fwd.hpp:17
filter_processor.hpp
c4::yml::QSCL
stored scalar was quoted
Definition: parser_state.hpp:30
c4::yml::ParseEngine::location_contents
csubstr location_contents(Location const &loc) const
Get the string starting at a particular location, to the end of the parsed source buffer...
_c4err
#define _c4err(...)
Definition: parse_engine.def.hpp:34
RYML_WITH_TAB_TOKENS_
#define RYML_WITH_TAB_TOKENS_(...)
Definition: parse_engine.def.hpp:46
c4::yml::version
csubstr version()
Definition: version.cpp:6
c4::basic_substring::first_not_of
size_t first_not_of(const C c) const
Definition: substr.hpp:993
c4::yml::read
ReadResult read(ConstNodeRef const &n, T *v)
Definition: node.hpp:2074
c4::yml::RSET
the (implicit) map being read is a !!set.
Definition: parser_state.hpp:31
_c4dbgnextline
#define _c4dbgnextline()
Definition: parse_engine.def.hpp:53
c4::yml::ParserFlag_t
int ParserFlag_t
data type for ParserState_e
Definition: parser_state.hpp:14
c4::yml::NDOC
no document mode. a document has ended and another has not started yet.
Definition: parser_state.hpp:33
c4::basic_substring< const char >::npos
Definition: substr.hpp:238
c4::yml::ParseEngine::operator=
ParseEngine & operator=(ParseEngine &&) noexcept
Definition: parse_engine.def.hpp:340
utf.hpp
utilities for UTF and Byte Order Mark
parse_engine.hpp
c4::yml::ParseEngine::ParseEngine
ParseEngine(EventHandler *evt_handler, ParserOptions const &opts={})
Definition: parse_engine.def.hpp:277