1 // pest. The Elegant Parser
2 // Copyright (c) 2018 Dragoș Tiselice
3 //
4 // Licensed under the Apache License, Version 2.0
5 // <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0> or the MIT
6 // license <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
7 // option. All files in the project carrying such notice may not be copied,
8 // modified, or distributed except according to those terms.
9
10 use std::path::PathBuf;
11
12 use proc_macro2::TokenStream;
13 use quote::{ToTokens, TokenStreamExt};
14 use syn::{self, Ident};
15
16 use pest::unicode::unicode_property_names;
17 use pest_meta::ast::*;
18 use pest_meta::optimizer::*;
19
20 use crate::docs::DocComment;
21 use crate::ParsedDerive;
22
generate( parsed_derive: ParsedDerive, paths: Vec<PathBuf>, rules: Vec<OptimizedRule>, defaults: Vec<&str>, doc_comment: &DocComment, include_grammar: bool, ) -> TokenStream23 pub(crate) fn generate(
24 parsed_derive: ParsedDerive,
25 paths: Vec<PathBuf>,
26 rules: Vec<OptimizedRule>,
27 defaults: Vec<&str>,
28 doc_comment: &DocComment,
29 include_grammar: bool,
30 ) -> TokenStream {
31 let uses_eoi = defaults.iter().any(|name| *name == "EOI");
32 let name = parsed_derive.name;
33 let builtins = generate_builtin_rules();
34 let include_fix = if include_grammar {
35 generate_include(&name, paths)
36 } else {
37 quote!()
38 };
39 let rule_enum = generate_enum(&rules, doc_comment, uses_eoi, parsed_derive.non_exhaustive);
40 let patterns = generate_patterns(&rules, uses_eoi);
41 let skip = generate_skip(&rules);
42
43 let mut rules: Vec<_> = rules.into_iter().map(generate_rule).collect();
44 rules.extend(builtins.into_iter().filter_map(|(builtin, tokens)| {
45 if defaults.contains(&builtin) {
46 Some(tokens)
47 } else {
48 None
49 }
50 }));
51
52 let (impl_generics, ty_generics, where_clause) = parsed_derive.generics.split_for_impl();
53
54 let result = result_type();
55
56 let parser_impl = quote! {
57 #[allow(clippy::all)]
58 impl #impl_generics ::pest::Parser<Rule> for #name #ty_generics #where_clause {
59 fn parse<'i>(
60 rule: Rule,
61 input: &'i str
62 ) -> #result<
63 ::pest::iterators::Pairs<'i, Rule>,
64 ::pest::error::Error<Rule>
65 > {
66 mod rules {
67 #![allow(clippy::upper_case_acronyms)]
68 pub mod hidden {
69 use super::super::Rule;
70 #skip
71 }
72
73 pub mod visible {
74 use super::super::Rule;
75 #( #rules )*
76 }
77
78 pub use self::visible::*;
79 }
80
81 ::pest::state(input, |state| {
82 match rule {
83 #patterns
84 }
85 })
86 }
87 }
88 };
89
90 quote! {
91 #include_fix
92 #rule_enum
93 #parser_impl
94 }
95 }
96
97 // Note: All builtin rules should be validated as pest builtins in meta/src/validator.rs.
98 // Some should also be keywords.
generate_builtin_rules() -> Vec<(&'static str, TokenStream)>99 fn generate_builtin_rules() -> Vec<(&'static str, TokenStream)> {
100 let mut builtins = Vec::new();
101
102 insert_builtin!(builtins, ANY, state.skip(1));
103 insert_builtin!(
104 builtins,
105 EOI,
106 state.rule(Rule::EOI, |state| state.end_of_input())
107 );
108 insert_builtin!(builtins, SOI, state.start_of_input());
109 insert_builtin!(builtins, PEEK, state.stack_peek());
110 insert_builtin!(builtins, PEEK_ALL, state.stack_match_peek());
111 insert_builtin!(builtins, POP, state.stack_pop());
112 insert_builtin!(builtins, POP_ALL, state.stack_match_pop());
113 insert_builtin!(builtins, DROP, state.stack_drop());
114
115 insert_builtin!(builtins, ASCII_DIGIT, state.match_range('0'..'9'));
116 insert_builtin!(builtins, ASCII_NONZERO_DIGIT, state.match_range('1'..'9'));
117 insert_builtin!(builtins, ASCII_BIN_DIGIT, state.match_range('0'..'1'));
118 insert_builtin!(builtins, ASCII_OCT_DIGIT, state.match_range('0'..'7'));
119 insert_builtin!(
120 builtins,
121 ASCII_HEX_DIGIT,
122 state
123 .match_range('0'..'9')
124 .or_else(|state| state.match_range('a'..'f'))
125 .or_else(|state| state.match_range('A'..'F'))
126 );
127 insert_builtin!(builtins, ASCII_ALPHA_LOWER, state.match_range('a'..'z'));
128 insert_builtin!(builtins, ASCII_ALPHA_UPPER, state.match_range('A'..'Z'));
129 insert_builtin!(
130 builtins,
131 ASCII_ALPHA,
132 state
133 .match_range('a'..'z')
134 .or_else(|state| state.match_range('A'..'Z'))
135 );
136 insert_builtin!(
137 builtins,
138 ASCII_ALPHANUMERIC,
139 state
140 .match_range('a'..'z')
141 .or_else(|state| state.match_range('A'..'Z'))
142 .or_else(|state| state.match_range('0'..'9'))
143 );
144 insert_builtin!(builtins, ASCII, state.match_range('\x00'..'\x7f'));
145 insert_builtin!(
146 builtins,
147 NEWLINE,
148 state
149 .match_string("\n")
150 .or_else(|state| state.match_string("\r\n"))
151 .or_else(|state| state.match_string("\r"))
152 );
153
154 let box_ty = box_type();
155
156 for property in unicode_property_names() {
157 let property_ident: Ident = syn::parse_str(property).unwrap();
158 // insert manually for #property substitution
159 builtins.push((property, quote! {
160 #[inline]
161 #[allow(dead_code, non_snake_case, unused_variables)]
162 fn #property_ident(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> {
163 state.match_char_by(::pest::unicode::#property_ident)
164 }
165 }));
166 }
167 builtins
168 }
169
170 /// Generate Rust `include_str!` for grammar files, then Cargo will watch changes in grammars.
generate_include(name: &Ident, paths: Vec<PathBuf>) -> TokenStream171 fn generate_include(name: &Ident, paths: Vec<PathBuf>) -> TokenStream {
172 let const_name = format_ident!("_PEST_GRAMMAR_{}", name);
173 // Need to make this relative to the current directory since the path to the file
174 // is derived from the CARGO_MANIFEST_DIR environment variable
175 let current_dir = std::env::current_dir().expect("Unable to get current directory");
176
177 let include_tokens = paths.iter().map(|path| {
178 let path = path.to_str().expect("non-Unicode path");
179
180 let relative_path = current_dir
181 .join(path)
182 .to_str()
183 .expect("path contains invalid unicode")
184 .to_string();
185
186 quote! {
187 include_str!(#relative_path)
188 }
189 });
190
191 let len = include_tokens.len();
192 quote! {
193 #[allow(non_upper_case_globals)]
194 const #const_name: [&'static str; #len] = [
195 #(#include_tokens),*
196 ];
197 }
198 }
199
generate_enum( rules: &[OptimizedRule], doc_comment: &DocComment, uses_eoi: bool, non_exhaustive: bool, ) -> TokenStream200 fn generate_enum(
201 rules: &[OptimizedRule],
202 doc_comment: &DocComment,
203 uses_eoi: bool,
204 non_exhaustive: bool,
205 ) -> TokenStream {
206 let rule_variants = rules.iter().map(|rule| {
207 let rule_name = format_ident!("r#{}", rule.name);
208
209 match doc_comment.line_docs.get(&rule.name) {
210 Some(doc) => quote! {
211 #[doc = #doc]
212 #rule_name
213 },
214 None => quote! {
215 #rule_name
216 },
217 }
218 });
219
220 let grammar_doc = &doc_comment.grammar_doc;
221 let mut result = quote! {
222 #[doc = #grammar_doc]
223 #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)]
224 #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
225 };
226 if non_exhaustive {
227 result.append_all(quote! {
228 #[non_exhaustive]
229 });
230 }
231 result.append_all(quote! {
232 pub enum Rule
233 });
234 if uses_eoi {
235 result.append_all(quote! {
236 {
237 #[doc = "End-of-input"]
238 EOI,
239 #( #rule_variants ),*
240 }
241 });
242 } else {
243 result.append_all(quote! {
244 {
245 #( #rule_variants ),*
246 }
247 })
248 };
249
250 let rules = rules.iter().map(|rule| {
251 let rule_name = format_ident!("r#{}", rule.name);
252 quote! { #rule_name }
253 });
254
255 result.append_all(quote! {
256 impl Rule {
257 pub fn all_rules() -> &'static[Rule] {
258 &[ #(Rule::#rules), * ]
259 }
260 }
261 });
262
263 result
264 }
265
generate_patterns(rules: &[OptimizedRule], uses_eoi: bool) -> TokenStream266 fn generate_patterns(rules: &[OptimizedRule], uses_eoi: bool) -> TokenStream {
267 let mut rules: Vec<TokenStream> = rules
268 .iter()
269 .map(|rule| {
270 let rule = format_ident!("r#{}", rule.name);
271
272 quote! {
273 Rule::#rule => rules::#rule(state)
274 }
275 })
276 .collect();
277
278 if uses_eoi {
279 rules.push(quote! {
280 Rule::EOI => rules::EOI(state)
281 });
282 }
283
284 quote! {
285 #( #rules ),*
286 }
287 }
288
generate_rule(rule: OptimizedRule) -> TokenStream289 fn generate_rule(rule: OptimizedRule) -> TokenStream {
290 let name = format_ident!("r#{}", rule.name);
291 let expr = if rule.ty == RuleType::Atomic || rule.ty == RuleType::CompoundAtomic {
292 generate_expr_atomic(rule.expr)
293 } else if rule.name == "WHITESPACE" || rule.name == "COMMENT" {
294 let atomic = generate_expr_atomic(rule.expr);
295
296 quote! {
297 state.atomic(::pest::Atomicity::Atomic, |state| {
298 #atomic
299 })
300 }
301 } else {
302 generate_expr(rule.expr)
303 };
304
305 let box_ty = box_type();
306
307 match rule.ty {
308 RuleType::Normal => quote! {
309 #[inline]
310 #[allow(non_snake_case, unused_variables)]
311 pub fn #name(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> {
312 state.rule(Rule::#name, |state| {
313 #expr
314 })
315 }
316 },
317 RuleType::Silent => quote! {
318 #[inline]
319 #[allow(non_snake_case, unused_variables)]
320 pub fn #name(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> {
321 #expr
322 }
323 },
324 RuleType::Atomic => quote! {
325 #[inline]
326 #[allow(non_snake_case, unused_variables)]
327 pub fn #name(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> {
328 state.rule(Rule::#name, |state| {
329 state.atomic(::pest::Atomicity::Atomic, |state| {
330 #expr
331 })
332 })
333 }
334 },
335 RuleType::CompoundAtomic => quote! {
336 #[inline]
337 #[allow(non_snake_case, unused_variables)]
338 pub fn #name(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> {
339 state.atomic(::pest::Atomicity::CompoundAtomic, |state| {
340 state.rule(Rule::#name, |state| {
341 #expr
342 })
343 })
344 }
345 },
346 RuleType::NonAtomic => quote! {
347 #[inline]
348 #[allow(non_snake_case, unused_variables)]
349 pub fn #name(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> {
350 state.atomic(::pest::Atomicity::NonAtomic, |state| {
351 state.rule(Rule::#name, |state| {
352 #expr
353 })
354 })
355 }
356 },
357 }
358 }
359
generate_skip(rules: &[OptimizedRule]) -> TokenStream360 fn generate_skip(rules: &[OptimizedRule]) -> TokenStream {
361 let whitespace = rules.iter().any(|rule| rule.name == "WHITESPACE");
362 let comment = rules.iter().any(|rule| rule.name == "COMMENT");
363
364 match (whitespace, comment) {
365 (false, false) => generate_rule!(skip, Ok(state)),
366 (true, false) => generate_rule!(
367 skip,
368 if state.atomicity() == ::pest::Atomicity::NonAtomic {
369 state.repeat(|state| super::visible::WHITESPACE(state))
370 } else {
371 Ok(state)
372 }
373 ),
374 (false, true) => generate_rule!(
375 skip,
376 if state.atomicity() == ::pest::Atomicity::NonAtomic {
377 state.repeat(|state| super::visible::COMMENT(state))
378 } else {
379 Ok(state)
380 }
381 ),
382 (true, true) => generate_rule!(
383 skip,
384 if state.atomicity() == ::pest::Atomicity::NonAtomic {
385 state.sequence(|state| {
386 state
387 .repeat(|state| super::visible::WHITESPACE(state))
388 .and_then(|state| {
389 state.repeat(|state| {
390 state.sequence(|state| {
391 super::visible::COMMENT(state).and_then(|state| {
392 state.repeat(|state| super::visible::WHITESPACE(state))
393 })
394 })
395 })
396 })
397 })
398 } else {
399 Ok(state)
400 }
401 ),
402 }
403 }
404
generate_expr(expr: OptimizedExpr) -> TokenStream405 fn generate_expr(expr: OptimizedExpr) -> TokenStream {
406 match expr {
407 OptimizedExpr::Str(string) => {
408 quote! {
409 state.match_string(#string)
410 }
411 }
412 OptimizedExpr::Insens(string) => {
413 quote! {
414 state.match_insensitive(#string)
415 }
416 }
417 OptimizedExpr::Range(start, end) => {
418 let start = start.chars().next().unwrap();
419 let end = end.chars().next().unwrap();
420
421 quote! {
422 state.match_range(#start..#end)
423 }
424 }
425 OptimizedExpr::Ident(ident) => {
426 let ident = format_ident!("r#{}", ident);
427 quote! { self::#ident(state) }
428 }
429 OptimizedExpr::PeekSlice(start, end_) => {
430 let end = QuoteOption(end_);
431 quote! {
432 state.stack_match_peek_slice(#start, #end, ::pest::MatchDir::BottomToTop)
433 }
434 }
435 OptimizedExpr::PosPred(expr) => {
436 let expr = generate_expr(*expr);
437
438 quote! {
439 state.lookahead(true, |state| {
440 #expr
441 })
442 }
443 }
444 OptimizedExpr::NegPred(expr) => {
445 let expr = generate_expr(*expr);
446
447 quote! {
448 state.lookahead(false, |state| {
449 #expr
450 })
451 }
452 }
453 OptimizedExpr::Seq(lhs, rhs) => {
454 let head = generate_expr(*lhs);
455 let mut tail = vec![];
456 let mut current = *rhs;
457
458 while let OptimizedExpr::Seq(lhs, rhs) = current {
459 tail.push(generate_expr(*lhs));
460 current = *rhs;
461 }
462 tail.push(generate_expr(current));
463
464 quote! {
465 state.sequence(|state| {
466 #head
467 #(
468 .and_then(|state| {
469 super::hidden::skip(state)
470 }).and_then(|state| {
471 #tail
472 })
473 )*
474 })
475 }
476 }
477 OptimizedExpr::Choice(lhs, rhs) => {
478 let head = generate_expr(*lhs);
479 let mut tail = vec![];
480 let mut current = *rhs;
481
482 while let OptimizedExpr::Choice(lhs, rhs) = current {
483 tail.push(generate_expr(*lhs));
484 current = *rhs;
485 }
486 tail.push(generate_expr(current));
487
488 quote! {
489 #head
490 #(
491 .or_else(|state| {
492 #tail
493 })
494 )*
495 }
496 }
497 OptimizedExpr::Opt(expr) => {
498 let expr = generate_expr(*expr);
499
500 quote! {
501 state.optional(|state| {
502 #expr
503 })
504 }
505 }
506 OptimizedExpr::Rep(expr) => {
507 let expr = generate_expr(*expr);
508
509 quote! {
510 state.sequence(|state| {
511 state.optional(|state| {
512 #expr.and_then(|state| {
513 state.repeat(|state| {
514 state.sequence(|state| {
515 super::hidden::skip(
516 state
517 ).and_then(|state| {
518 #expr
519 })
520 })
521 })
522 })
523 })
524 })
525 }
526 }
527 #[cfg(feature = "grammar-extras")]
528 OptimizedExpr::RepOnce(expr) => {
529 let expr = generate_expr(*expr);
530
531 quote! {
532 state.sequence(|state| {
533 #expr.and_then(|state| {
534 state.repeat(|state| {
535 state.sequence(|state| {
536 super::hidden::skip(
537 state
538 ).and_then(|state| {
539 #expr
540 })
541 })
542 })
543 })
544 })
545 }
546 }
547 OptimizedExpr::Skip(strings) => {
548 quote! {
549 let strings = [#(#strings),*];
550
551 state.skip_until(&strings)
552 }
553 }
554 OptimizedExpr::Push(expr) => {
555 let expr = generate_expr(*expr);
556
557 quote! {
558 state.stack_push(|state| #expr)
559 }
560 }
561 OptimizedExpr::RestoreOnErr(expr) => {
562 let expr = generate_expr(*expr);
563
564 quote! {
565 state.restore_on_err(|state| #expr)
566 }
567 }
568 #[cfg(feature = "grammar-extras")]
569 OptimizedExpr::NodeTag(expr, tag) => {
570 let expr = generate_expr(*expr);
571 quote! {
572 #expr.and_then(|state| state.tag_node(#tag))
573 }
574 }
575 }
576 }
577
generate_expr_atomic(expr: OptimizedExpr) -> TokenStream578 fn generate_expr_atomic(expr: OptimizedExpr) -> TokenStream {
579 match expr {
580 OptimizedExpr::Str(string) => {
581 quote! {
582 state.match_string(#string)
583 }
584 }
585 OptimizedExpr::Insens(string) => {
586 quote! {
587 state.match_insensitive(#string)
588 }
589 }
590 OptimizedExpr::Range(start, end) => {
591 let start = start.chars().next().unwrap();
592 let end = end.chars().next().unwrap();
593
594 quote! {
595 state.match_range(#start..#end)
596 }
597 }
598 OptimizedExpr::Ident(ident) => {
599 let ident = format_ident!("r#{}", ident);
600 quote! { self::#ident(state) }
601 }
602 OptimizedExpr::PeekSlice(start, end_) => {
603 let end = QuoteOption(end_);
604 quote! {
605 state.stack_match_peek_slice(#start, #end, ::pest::MatchDir::BottomToTop)
606 }
607 }
608 OptimizedExpr::PosPred(expr) => {
609 let expr = generate_expr_atomic(*expr);
610
611 quote! {
612 state.lookahead(true, |state| {
613 #expr
614 })
615 }
616 }
617 OptimizedExpr::NegPred(expr) => {
618 let expr = generate_expr_atomic(*expr);
619
620 quote! {
621 state.lookahead(false, |state| {
622 #expr
623 })
624 }
625 }
626 OptimizedExpr::Seq(lhs, rhs) => {
627 let head = generate_expr_atomic(*lhs);
628 let mut tail = vec![];
629 let mut current = *rhs;
630
631 while let OptimizedExpr::Seq(lhs, rhs) = current {
632 tail.push(generate_expr_atomic(*lhs));
633 current = *rhs;
634 }
635 tail.push(generate_expr_atomic(current));
636
637 quote! {
638 state.sequence(|state| {
639 #head
640 #(
641 .and_then(|state| {
642 #tail
643 })
644 )*
645 })
646 }
647 }
648 OptimizedExpr::Choice(lhs, rhs) => {
649 let head = generate_expr_atomic(*lhs);
650 let mut tail = vec![];
651 let mut current = *rhs;
652
653 while let OptimizedExpr::Choice(lhs, rhs) = current {
654 tail.push(generate_expr_atomic(*lhs));
655 current = *rhs;
656 }
657 tail.push(generate_expr_atomic(current));
658
659 quote! {
660 #head
661 #(
662 .or_else(|state| {
663 #tail
664 })
665 )*
666 }
667 }
668 OptimizedExpr::Opt(expr) => {
669 let expr = generate_expr_atomic(*expr);
670
671 quote! {
672 state.optional(|state| {
673 #expr
674 })
675 }
676 }
677 OptimizedExpr::Rep(expr) => {
678 let expr = generate_expr_atomic(*expr);
679
680 quote! {
681 state.repeat(|state| {
682 #expr
683 })
684 }
685 }
686 #[cfg(feature = "grammar-extras")]
687 OptimizedExpr::RepOnce(expr) => {
688 let expr = generate_expr_atomic(*expr);
689
690 quote! {
691 state.sequence(|state| {
692 #expr.and_then(|state| {
693 state.repeat(|state| {
694 state.sequence(|state| {
695 #expr
696 })
697 })
698 })
699 })
700 }
701 }
702 OptimizedExpr::Skip(strings) => {
703 quote! {
704 let strings = [#(#strings),*];
705
706 state.skip_until(&strings)
707 }
708 }
709 OptimizedExpr::Push(expr) => {
710 let expr = generate_expr_atomic(*expr);
711
712 quote! {
713 state.stack_push(|state| #expr)
714 }
715 }
716 OptimizedExpr::RestoreOnErr(expr) => {
717 let expr = generate_expr_atomic(*expr);
718
719 quote! {
720 state.restore_on_err(|state| #expr)
721 }
722 }
723 #[cfg(feature = "grammar-extras")]
724 OptimizedExpr::NodeTag(expr, tag) => {
725 let expr = generate_expr_atomic(*expr);
726 quote! {
727 #expr.and_then(|state| state.tag_node(#tag))
728 }
729 }
730 }
731 }
732
733 struct QuoteOption<T>(Option<T>);
734
735 impl<T: ToTokens> ToTokens for QuoteOption<T> {
to_tokens(&self, tokens: &mut TokenStream)736 fn to_tokens(&self, tokens: &mut TokenStream) {
737 let option = option_type();
738 tokens.append_all(match self.0 {
739 Some(ref t) => quote! { #option::Some(#t) },
740 None => quote! { #option::None },
741 });
742 }
743 }
744
box_type() -> TokenStream745 fn box_type() -> TokenStream {
746 #[cfg(feature = "std")]
747 quote! { ::std::boxed::Box }
748
749 #[cfg(not(feature = "std"))]
750 quote! { ::alloc::boxed::Box }
751 }
752
result_type() -> TokenStream753 fn result_type() -> TokenStream {
754 #[cfg(feature = "std")]
755 quote! { ::std::result::Result }
756
757 #[cfg(not(feature = "std"))]
758 quote! { ::core::result::Result }
759 }
760
option_type() -> TokenStream761 fn option_type() -> TokenStream {
762 #[cfg(feature = "std")]
763 quote! { ::std::option::Option }
764
765 #[cfg(not(feature = "std"))]
766 quote! { ::core::option::Option }
767 }
768
769 #[cfg(test)]
770 mod tests {
771 use super::*;
772
773 use proc_macro2::Span;
774 use std::collections::HashMap;
775 use syn::Generics;
776
777 #[test]
rule_enum_simple()778 fn rule_enum_simple() {
779 let rules = vec![OptimizedRule {
780 name: "f".to_owned(),
781 ty: RuleType::Normal,
782 expr: OptimizedExpr::Ident("g".to_owned()),
783 }];
784
785 let mut line_docs = HashMap::new();
786 line_docs.insert("f".to_owned(), "This is rule comment".to_owned());
787
788 let doc_comment = &DocComment {
789 grammar_doc: "Rule doc\nhello".to_owned(),
790 line_docs,
791 };
792
793 assert_eq!(
794 generate_enum(&rules, doc_comment, false, false).to_string(),
795 quote! {
796 #[doc = "Rule doc\nhello"]
797 #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)]
798 #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
799 pub enum Rule {
800 #[doc = "This is rule comment"]
801 r#f
802 }
803 impl Rule {
804 pub fn all_rules() -> &'static [Rule] {
805 &[Rule::r#f]
806 }
807 }
808 }
809 .to_string()
810 );
811 }
812
813 #[test]
sequence()814 fn sequence() {
815 let expr = OptimizedExpr::Seq(
816 Box::new(OptimizedExpr::Str("a".to_owned())),
817 Box::new(OptimizedExpr::Seq(
818 Box::new(OptimizedExpr::Str("b".to_owned())),
819 Box::new(OptimizedExpr::Seq(
820 Box::new(OptimizedExpr::Str("c".to_owned())),
821 Box::new(OptimizedExpr::Str("d".to_owned())),
822 )),
823 )),
824 );
825
826 assert_eq!(
827 generate_expr(expr).to_string(),
828 quote! {
829 state.sequence(|state| {
830 state.match_string("a").and_then(|state| {
831 super::hidden::skip(state)
832 }).and_then(|state| {
833 state.match_string("b")
834 }).and_then(|state| {
835 super::hidden::skip(state)
836 }).and_then(|state| {
837 state.match_string("c")
838 }).and_then(|state| {
839 super::hidden::skip(state)
840 }).and_then(|state| {
841 state.match_string("d")
842 })
843 })
844 }
845 .to_string()
846 );
847 }
848
849 #[test]
sequence_atomic()850 fn sequence_atomic() {
851 let expr = OptimizedExpr::Seq(
852 Box::new(OptimizedExpr::Str("a".to_owned())),
853 Box::new(OptimizedExpr::Seq(
854 Box::new(OptimizedExpr::Str("b".to_owned())),
855 Box::new(OptimizedExpr::Seq(
856 Box::new(OptimizedExpr::Str("c".to_owned())),
857 Box::new(OptimizedExpr::Str("d".to_owned())),
858 )),
859 )),
860 );
861
862 assert_eq!(
863 generate_expr_atomic(expr).to_string(),
864 quote! {
865 state.sequence(|state| {
866 state.match_string("a").and_then(|state| {
867 state.match_string("b")
868 }).and_then(|state| {
869 state.match_string("c")
870 }).and_then(|state| {
871 state.match_string("d")
872 })
873 })
874 }
875 .to_string()
876 );
877 }
878
879 #[test]
choice()880 fn choice() {
881 let expr = OptimizedExpr::Choice(
882 Box::new(OptimizedExpr::Str("a".to_owned())),
883 Box::new(OptimizedExpr::Choice(
884 Box::new(OptimizedExpr::Str("b".to_owned())),
885 Box::new(OptimizedExpr::Choice(
886 Box::new(OptimizedExpr::Str("c".to_owned())),
887 Box::new(OptimizedExpr::Str("d".to_owned())),
888 )),
889 )),
890 );
891
892 assert_eq!(
893 generate_expr(expr).to_string(),
894 quote! {
895 state.match_string("a").or_else(|state| {
896 state.match_string("b")
897 }).or_else(|state| {
898 state.match_string("c")
899 }).or_else(|state| {
900 state.match_string("d")
901 })
902 }
903 .to_string()
904 );
905 }
906
907 #[test]
choice_atomic()908 fn choice_atomic() {
909 let expr = OptimizedExpr::Choice(
910 Box::new(OptimizedExpr::Str("a".to_owned())),
911 Box::new(OptimizedExpr::Choice(
912 Box::new(OptimizedExpr::Str("b".to_owned())),
913 Box::new(OptimizedExpr::Choice(
914 Box::new(OptimizedExpr::Str("c".to_owned())),
915 Box::new(OptimizedExpr::Str("d".to_owned())),
916 )),
917 )),
918 );
919
920 assert_eq!(
921 generate_expr_atomic(expr).to_string(),
922 quote! {
923 state.match_string("a").or_else(|state| {
924 state.match_string("b")
925 }).or_else(|state| {
926 state.match_string("c")
927 }).or_else(|state| {
928 state.match_string("d")
929 })
930 }
931 .to_string()
932 );
933 }
934
935 #[test]
skip()936 fn skip() {
937 let expr = OptimizedExpr::Skip(vec!["a".to_owned(), "b".to_owned()]);
938
939 assert_eq!(
940 generate_expr_atomic(expr).to_string(),
941 quote! {
942 let strings = ["a", "b"];
943
944 state.skip_until(&strings)
945 }
946 .to_string()
947 );
948 }
949
950 #[test]
expr_complex()951 fn expr_complex() {
952 let expr = OptimizedExpr::Choice(
953 Box::new(OptimizedExpr::Ident("a".to_owned())),
954 Box::new(OptimizedExpr::Seq(
955 Box::new(OptimizedExpr::Range("a".to_owned(), "b".to_owned())),
956 Box::new(OptimizedExpr::Seq(
957 Box::new(OptimizedExpr::NegPred(Box::new(OptimizedExpr::Rep(
958 Box::new(OptimizedExpr::Insens("b".to_owned())),
959 )))),
960 Box::new(OptimizedExpr::PosPred(Box::new(OptimizedExpr::Opt(
961 Box::new(OptimizedExpr::Rep(Box::new(OptimizedExpr::Choice(
962 Box::new(OptimizedExpr::Str("c".to_owned())),
963 Box::new(OptimizedExpr::Str("d".to_owned())),
964 )))),
965 )))),
966 )),
967 )),
968 );
969
970 let sequence = quote! {
971 state.sequence(|state| {
972 super::hidden::skip(state).and_then(
973 |state| {
974 state.match_insensitive("b")
975 }
976 )
977 })
978 };
979 let repeat = quote! {
980 state.repeat(|state| {
981 state.sequence(|state| {
982 super::hidden::skip(state).and_then(|state| {
983 state.match_string("c")
984 .or_else(|state| {
985 state.match_string("d")
986 })
987 })
988 })
989 })
990 };
991 assert_eq!(
992 generate_expr(expr).to_string(),
993 quote! {
994 self::r#a(state).or_else(|state| {
995 state.sequence(|state| {
996 state.match_range('a'..'b').and_then(|state| {
997 super::hidden::skip(state)
998 }).and_then(|state| {
999 state.lookahead(false, |state| {
1000 state.sequence(|state| {
1001 state.optional(|state| {
1002 state.match_insensitive(
1003 "b"
1004 ).and_then(|state| {
1005 state.repeat(|state| {
1006 #sequence
1007 })
1008 })
1009 })
1010 })
1011 })
1012 }).and_then(|state| {
1013 super::hidden::skip(state)
1014 }).and_then(|state| {
1015 state.lookahead(true, |state| {
1016 state.optional(|state| {
1017 state.sequence(|state| {
1018 state.optional(|state| {
1019 state.match_string("c")
1020 .or_else(|state| {
1021 state.match_string("d")
1022 }).and_then(|state| {
1023 #repeat
1024 })
1025 })
1026 })
1027 })
1028 })
1029 })
1030 })
1031 })
1032 }
1033 .to_string()
1034 );
1035 }
1036
1037 #[test]
expr_complex_atomic()1038 fn expr_complex_atomic() {
1039 let expr = OptimizedExpr::Choice(
1040 Box::new(OptimizedExpr::Ident("a".to_owned())),
1041 Box::new(OptimizedExpr::Seq(
1042 Box::new(OptimizedExpr::Range("a".to_owned(), "b".to_owned())),
1043 Box::new(OptimizedExpr::Seq(
1044 Box::new(OptimizedExpr::NegPred(Box::new(OptimizedExpr::Rep(
1045 Box::new(OptimizedExpr::Insens("b".to_owned())),
1046 )))),
1047 Box::new(OptimizedExpr::PosPred(Box::new(OptimizedExpr::Opt(
1048 Box::new(OptimizedExpr::Rep(Box::new(OptimizedExpr::Choice(
1049 Box::new(OptimizedExpr::Str("c".to_owned())),
1050 Box::new(OptimizedExpr::Str("d".to_owned())),
1051 )))),
1052 )))),
1053 )),
1054 )),
1055 );
1056
1057 assert_eq!(
1058 generate_expr_atomic(expr).to_string(),
1059 quote! {
1060 self::r#a(state).or_else(|state| {
1061 state.sequence(|state| {
1062 state.match_range('a'..'b').and_then(|state| {
1063 state.lookahead(false, |state| {
1064 state.repeat(|state| {
1065 state.match_insensitive("b")
1066 })
1067 })
1068 }).and_then(|state| {
1069 state.lookahead(true, |state| {
1070 state.optional(|state| {
1071 state.repeat(|state| {
1072 state.match_string("c")
1073 .or_else(|state| {
1074 state.match_string("d")
1075 })
1076 })
1077 })
1078 })
1079 })
1080 })
1081 })
1082 }
1083 .to_string()
1084 );
1085 }
1086
1087 #[test]
test_generate_complete()1088 fn test_generate_complete() {
1089 let name = Ident::new("MyParser", Span::call_site());
1090 let generics = Generics::default();
1091
1092 let rules = vec![
1093 OptimizedRule {
1094 name: "a".to_owned(),
1095 ty: RuleType::Silent,
1096 expr: OptimizedExpr::Str("b".to_owned()),
1097 },
1098 OptimizedRule {
1099 name: "if".to_owned(),
1100 ty: RuleType::Silent,
1101 expr: OptimizedExpr::Ident("a".to_owned()),
1102 },
1103 ];
1104
1105 let mut line_docs = HashMap::new();
1106 line_docs.insert("if".to_owned(), "If statement".to_owned());
1107
1108 let doc_comment = &DocComment {
1109 line_docs,
1110 grammar_doc: "This is Rule doc\nThis is second line".to_owned(),
1111 };
1112
1113 let defaults = vec!["ANY"];
1114 let result = result_type();
1115 let box_ty = box_type();
1116 let current_dir = std::env::current_dir().expect("Unable to get current directory");
1117
1118 let base_path = current_dir.join("base.pest").to_str().unwrap().to_string();
1119 let test_path = current_dir.join("test.pest").to_str().unwrap().to_string();
1120 let parsed_derive = ParsedDerive {
1121 name,
1122 generics,
1123 non_exhaustive: false,
1124 };
1125 assert_eq!(
1126 generate(parsed_derive, vec![PathBuf::from("base.pest"), PathBuf::from("test.pest")], rules, defaults, doc_comment, true).to_string(),
1127 quote! {
1128 #[allow(non_upper_case_globals)]
1129 const _PEST_GRAMMAR_MyParser: [&'static str; 2usize] = [include_str!(#base_path), include_str!(#test_path)];
1130
1131 #[doc = "This is Rule doc\nThis is second line"]
1132 #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)]
1133 #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
1134 pub enum Rule {
1135 r#a,
1136 #[doc = "If statement"]
1137 r#if
1138 }
1139 impl Rule {
1140 pub fn all_rules() -> &'static [Rule] {
1141 &[Rule::r#a, Rule::r#if]
1142 }
1143 }
1144
1145 #[allow(clippy::all)]
1146 impl ::pest::Parser<Rule> for MyParser {
1147 fn parse<'i>(
1148 rule: Rule,
1149 input: &'i str
1150 ) -> #result<
1151 ::pest::iterators::Pairs<'i, Rule>,
1152 ::pest::error::Error<Rule>
1153 > {
1154 mod rules {
1155 #![allow(clippy::upper_case_acronyms)]
1156 pub mod hidden {
1157 use super::super::Rule;
1158
1159 #[inline]
1160 #[allow(dead_code, non_snake_case, unused_variables)]
1161 pub fn skip(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> {
1162 Ok(state)
1163 }
1164 }
1165
1166 pub mod visible {
1167 use super::super::Rule;
1168
1169 #[inline]
1170 #[allow(non_snake_case, unused_variables)]
1171 pub fn r#a(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> {
1172 state.match_string("b")
1173 }
1174
1175 #[inline]
1176 #[allow(non_snake_case, unused_variables)]
1177 pub fn r#if(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> {
1178 self::r#a(state)
1179 }
1180
1181 #[inline]
1182 #[allow(dead_code, non_snake_case, unused_variables)]
1183 pub fn ANY(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> {
1184 state.skip(1)
1185 }
1186 }
1187
1188 pub use self::visible::*;
1189 }
1190
1191 ::pest::state(input, |state| {
1192 match rule {
1193 Rule::r#a => rules::r#a(state),
1194 Rule::r#if => rules::r#if(state)
1195 }
1196 })
1197 }
1198 }
1199 }.to_string()
1200 );
1201 }
1202 }
1203