tinytemplate/

compiler.rs

#![allow(deprecated)]

/// The compiler module houses the code which parses and compiles templates. TinyTemplate implements
/// a simple bytecode interpreter (see the [instruction] module for more details) to render templates.
/// The [`TemplateCompiler`](struct.TemplateCompiler.html) struct is responsible for parsing the
/// template strings and generating the appropriate bytecode instructions.
use error::Error::*;
use error::{get_offset, Error, Result};
use instruction::{Instruction, Path, PathStep};

/// The end point of a branch or goto instruction is not known.
const UNKNOWN: usize = ::std::usize::MAX;

/// The compiler keeps a stack of the open blocks so that it can ensure that blocks are closed in
/// the right order. The Block type is a simple enumeration of the kinds of blocks that could be
/// open. It may contain the instruction index corresponding to the start of the block.
enum Block {
    Branch(usize),
    For(usize),
    With,
}

/// List of the known @-keywords so that we can error if the user spells them wrong.
static KNOWN_KEYWORDS: [&str; 4] = ["@index", "@first", "@last", "@root"];

/// The TemplateCompiler struct is responsible for parsing a template string and generating bytecode
/// instructions based on it. The parser is a simple hand-written pattern-matching parser with no
/// recursion, which makes it relatively easy to read.
pub(crate) struct TemplateCompiler<'template> {
    original_text: &'template str,
    remaining_text: &'template str,
    instructions: Vec<Instruction<'template>>,
    block_stack: Vec<(&'template str, Block)>,

    /// When we see a `{foo -}` or similar, we need to remember to left-trim the next text block we
    /// encounter.
    trim_next: bool,
}
impl<'template> TemplateCompiler<'template> {
    /// Create a new template compiler to parse and compile the given template.
    pub fn new(text: &'template str) -> TemplateCompiler<'template> {
        TemplateCompiler {
            original_text: text,
            remaining_text: text,
            instructions: vec![],
            block_stack: vec![],
            trim_next: false,
        }
    }

    /// Consume the template compiler to parse the template and return the generated bytecode.
    pub fn compile(mut self) -> Result<Vec<Instruction<'template>>> {
        while !self.remaining_text.is_empty() {
            // Comment, denoted by {# comment text #}
            if self.remaining_text.starts_with("{#") {
                self.trim_next = false;

                let tag = self.consume_tag("#}")?;
                let comment = tag[2..(tag.len() - 2)].trim();
                if comment.starts_with('-') {
                    self.trim_last_whitespace();
                }
                if comment.ends_with('-') {
                    self.trim_next_whitespace();
                }
            // Block tag. Block tags are wrapped in {{ }} and always have one word at the start
            // to identify which kind of tag it is. Depending on the tag type there may be more.
            } else if self.remaining_text.starts_with("{{") {
                self.trim_next = false;

                let (discriminant, rest) = self.consume_block()?;
                match discriminant {
                    "if" => {
                        let (path, negated) = if rest.starts_with("not") {
                            (self.parse_path(&rest[4..])?, true)
                        } else {
                            (self.parse_path(rest)?, false)
                        };
                        self.block_stack
                            .push((discriminant, Block::Branch(self.instructions.len())));
                        self.instructions
                            .push(Instruction::Branch(path, !negated, UNKNOWN));
                    }
                    "else" => {
                        self.expect_empty(rest)?;
                        let num_instructions = self.instructions.len() + 1;
                        self.close_branch(num_instructions, discriminant)?;
                        self.block_stack
                            .push((discriminant, Block::Branch(self.instructions.len())));
                        self.instructions.push(Instruction::Goto(UNKNOWN))
                    }
                    "endif" => {
                        self.expect_empty(rest)?;
                        let num_instructions = self.instructions.len();
                        self.close_branch(num_instructions, discriminant)?;
                    }
                    "with" => {
                        let (path, name) = self.parse_with(rest)?;
                        let instruction = Instruction::PushNamedContext(path, name);
                        self.instructions.push(instruction);
                        self.block_stack.push((discriminant, Block::With));
                    }
                    "endwith" => {
                        self.expect_empty(rest)?;
                        if let Some((_, Block::With)) = self.block_stack.pop() {
                            self.instructions.push(Instruction::PopContext)
                        } else {
                            return Err(self.parse_error(
                                discriminant,
                                "Found a closing endwith that doesn't match with a preceeding with.".to_string()
                            ));
                        }
                    }
                    "for" => {
                        let (path, name) = self.parse_for(rest)?;
                        self.instructions
                            .push(Instruction::PushIterationContext(path, name));
                        self.block_stack
                            .push((discriminant, Block::For(self.instructions.len())));
                        self.instructions.push(Instruction::Iterate(UNKNOWN));
                    }
                    "endfor" => {
                        self.expect_empty(rest)?;
                        let num_instructions = self.instructions.len() + 1;
                        let goto_target = self.close_for(num_instructions, discriminant)?;
                        self.instructions.push(Instruction::Goto(goto_target));
                        self.instructions.push(Instruction::PopContext);
                    }
                    "call" => {
                        let (name, path) = self.parse_call(rest)?;
                        self.instructions.push(Instruction::Call(name, path));
                    }
                    _ => {
                        return Err(self.parse_error(
                            discriminant,
                            format!("Unknown block type '{}'", discriminant),
                        ));
                    }
                }
            // Values, of the form { dotted.path.to.value.in.context }
            // Note that it is not (currently) possible to escape curly braces in the templates to
            // prevent them from being interpreted as values.
            } else if self.remaining_text.starts_with('{') {
                self.trim_next = false;

                let (path, name) = self.consume_value()?;
                let instruction = match name {
                    Some(name) => Instruction::FormattedValue(path, name),
                    None => Instruction::Value(path),
                };
                self.instructions.push(instruction);
            // All other text - just consume characters until we see a {
            } else {
                let mut escaped = false;
                loop {
                    let mut text = self.consume_text(escaped);
                    if self.trim_next {
                        text = text.trim_left();
                        self.trim_next = false;
                    }
                    escaped = text.ends_with('\\');
                    if escaped {
                        text = &text[0..(text.len() - 1)];
                    }
                    self.instructions.push(Instruction::Literal(text));

                    if !escaped {
                        break;
                    }
                }
            }
        }

        if let Some((text, _)) = self.block_stack.pop() {
            return Err(self.parse_error(
                text,
                "Expected block-closing tag, but reached the end of input.".to_string(),
            ));
        }

        Ok(self.instructions)
    }

    /// Splits a string into a list of named segments which can later be used to look up values in the
    /// context.
    fn parse_path(&self, text: &'template str) -> Result<Path<'template>> {
        if !text.starts_with('@') {
            Ok(text
                .split('.')
                .map(|s| match s.parse::<usize>() {
                    Ok(n) => PathStep::Index(s, n),
                    Err(_) => PathStep::Name(s),
                })
                .collect::<Vec<_>>())
        } else if KNOWN_KEYWORDS.iter().any(|k| *k == text) {
            Ok(vec![PathStep::Name(text)])
        } else {
            Err(self.parse_error(text, format!("Invalid keyword name '{}'", text)))
        }
    }

    /// Finds the line number and column where an error occurred. Location is the substring of
    /// self.original_text where the error was found, and msg is the error message.
    fn parse_error(&self, location: &str, msg: String) -> Error {
        let (line, column) = get_offset(self.original_text, location);
        ParseError { msg, line, column }
    }

    /// Tags which should have no text after the discriminant use this to raise an error if
    /// text is found.
    fn expect_empty(&self, text: &str) -> Result<()> {
        if text.is_empty() {
            Ok(())
        } else {
            Err(self.parse_error(text, format!("Unexpected text '{}'", text)))
        }
    }

    /// Close the branch that is on top of the block stack by setting its target instruction
    /// and popping it from the stack. Returns an error if the top of the block stack is not a
    /// branch.
    fn close_branch(&mut self, new_target: usize, discriminant: &str) -> Result<()> {
        let branch_block = self.block_stack.pop();
        if let Some((_, Block::Branch(index))) = branch_block {
            match &mut self.instructions[index] {
                Instruction::Branch(_, _, target) => {
                    *target = new_target;
                    Ok(())
                }
                Instruction::Goto(target) => {
                    *target = new_target;
                    Ok(())
                }
                _ => panic!(),
            }
        } else {
            Err(self.parse_error(
                discriminant,
                "Found a closing endif or else which doesn't match with a preceding if."
                    .to_string(),
            ))
        }
    }

    /// Close the for loop that is on top of the block stack by setting its target instruction and
    /// popping it from the stack. Returns an error if the top of the stack is not a for loop.
    /// Returns the index of the loop's Iterate instruction for further processing.
    fn close_for(&mut self, new_target: usize, discriminant: &str) -> Result<usize> {
        let branch_block = self.block_stack.pop();
        if let Some((_, Block::For(index))) = branch_block {
            match &mut self.instructions[index] {
                Instruction::Iterate(target) => {
                    *target = new_target;
                    Ok(index)
                }
                _ => panic!(),
            }
        } else {
            Err(self.parse_error(
                discriminant,
                "Found a closing endfor which doesn't match with a preceding for.".to_string(),
            ))
        }
    }

    /// Advance the cursor to the next { and return the consumed text. If `escaped` is true, skips
    /// a { at the start of the text.
    fn consume_text(&mut self, escaped: bool) -> &'template str {
        let search_substr = if escaped {
            &self.remaining_text[1..]
        } else {
            self.remaining_text
        };

        let mut position = search_substr
            .find('{')
            .unwrap_or_else(|| search_substr.len());
        if escaped {
            position += 1;
        }

        let (text, remaining) = self.remaining_text.split_at(position);
        self.remaining_text = remaining;
        text
    }

    /// Advance the cursor to the end of the value tag and return the value's path and optional
    /// formatter name.
    fn consume_value(&mut self) -> Result<(Path<'template>, Option<&'template str>)> {
        let tag = self.consume_tag("}")?;
        let mut tag = tag[1..(tag.len() - 1)].trim();
        if tag.starts_with('-') {
            tag = tag[1..].trim();
            self.trim_last_whitespace();
        }
        if tag.ends_with('-') {
            tag = tag[0..tag.len() - 1].trim();
            self.trim_next_whitespace();
        }

        if let Some(index) = tag.find('|') {
            let (path_str, name_str) = tag.split_at(index);
            let name = name_str[1..].trim();
            let path = self.parse_path(path_str.trim())?;
            Ok((path, Some(name)))
        } else {
            Ok((self.parse_path(tag)?, None))
        }
    }

    /// Right-trim whitespace from the last text block we parsed.
    fn trim_last_whitespace(&mut self) {
        if let Some(Instruction::Literal(text)) = self.instructions.last_mut() {
            *text = text.trim_right();
        }
    }

    /// Make a note to left-trim whitespace from the next text block we parse.
    fn trim_next_whitespace(&mut self) {
        self.trim_next = true;
    }

    /// Advance the cursor to the end of the current block tag and return the discriminant substring
    /// and the rest of the text in the tag. Also handles trimming whitespace where needed.
    fn consume_block(&mut self) -> Result<(&'template str, &'template str)> {
        let tag = self.consume_tag("}}")?;
        let mut block = tag[2..(tag.len() - 2)].trim();
        if block.starts_with('-') {
            block = block[1..].trim();
            self.trim_last_whitespace();
        }
        if block.ends_with('-') {
            block = block[0..block.len() - 1].trim();
            self.trim_next_whitespace();
        }
        let discriminant = block.split_whitespace().next().unwrap_or(block);
        let rest = block[discriminant.len()..].trim();
        Ok((discriminant, rest))
    }

    /// Advance the cursor to after the given expected_close string and return the text in between
    /// (including the expected_close characters), or return an error message if we reach the end
    /// of a line of text without finding it.
    fn consume_tag(&mut self, expected_close: &str) -> Result<&'template str> {
        if let Some(line) = self.remaining_text.lines().next() {
            if let Some(pos) = line.find(expected_close) {
                let (tag, remaining) = self.remaining_text.split_at(pos + expected_close.len());
                self.remaining_text = remaining;
                Ok(tag)
            } else {
                Err(self.parse_error(
                    line,
                    format!(
                        "Expected a closing '{}' but found end-of-line instead.",
                        expected_close
                    ),
                ))
            }
        } else {
            Err(self.parse_error(
                self.remaining_text,
                format!(
                    "Expected a closing '{}' but found end-of-text instead.",
                    expected_close
                ),
            ))
        }
    }

    /// Parse a with tag to separate the value path from the (optional) name.
    fn parse_with(&self, with_text: &'template str) -> Result<(Path<'template>, &'template str)> {
        if let Some(index) = with_text.find(" as ") {
            let (path_str, name_str) = with_text.split_at(index);
            let path = self.parse_path(path_str.trim())?;
            let name = name_str[" as ".len()..].trim();
            Ok((path, name))
        } else {
            Err(self.parse_error(
                with_text,
                format!(
                    "Expected 'as <path>' in with block, but found \"{}\" instead",
                    with_text
                ),
            ))
        }
    }

    /// Parse a for tag to separate the value path from the name.
    fn parse_for(&self, for_text: &'template str) -> Result<(Path<'template>, &'template str)> {
        if let Some(index) = for_text.find(" in ") {
            let (name_str, path_str) = for_text.split_at(index);
            let name = name_str.trim();
            let path = self.parse_path(path_str[" in ".len()..].trim())?;
            Ok((path, name))
        } else {
            Err(self.parse_error(
                for_text,
                format!("Unable to parse for block text '{}'", for_text),
            ))
        }
    }

    /// Parse a call tag to separate the template name and context value.
    fn parse_call(&self, call_text: &'template str) -> Result<(&'template str, Path<'template>)> {
        if let Some(index) = call_text.find(" with ") {
            let (name_str, path_str) = call_text.split_at(index);
            let name = name_str.trim();
            let path = self.parse_path(path_str[" with ".len()..].trim())?;
            Ok((name, path))
        } else {
            Err(self.parse_error(
                call_text,
                format!("Unable to parse call block text '{}'", call_text),
            ))
        }
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use instruction::Instruction::*;

    fn compile(text: &'static str) -> Result<Vec<Instruction<'static>>> {
        TemplateCompiler::new(text).compile()
    }

    #[test]
    fn test_compile_literal() {
        let text = "Test String";
        let instructions = compile(text).unwrap();
        assert_eq!(1, instructions.len());
        assert_eq!(&Literal(text), &instructions[0]);
    }

    #[test]
    fn test_compile_value() {
        let text = "{ foobar }";
        let instructions = compile(text).unwrap();
        assert_eq!(1, instructions.len());
        assert_eq!(&Value(vec![PathStep::Name("foobar")]), &instructions[0]);
    }

    #[test]
    fn test_compile_value_with_formatter() {
        let text = "{ foobar | my_formatter }";
        let instructions = compile(text).unwrap();
        assert_eq!(1, instructions.len());
        assert_eq!(
            &FormattedValue(vec![PathStep::Name("foobar")], "my_formatter"),
            &instructions[0]
        );
    }

    #[test]
    fn test_dotted_path() {
        let text = "{ foo.bar }";
        let instructions = compile(text).unwrap();
        assert_eq!(1, instructions.len());
        assert_eq!(
            &Value(vec![PathStep::Name("foo"), PathStep::Name("bar")]),
            &instructions[0]
        );
    }

    #[test]
    fn test_indexed_path() {
        let text = "{ foo.0.bar }";
        let instructions = compile(text).unwrap();
        assert_eq!(1, instructions.len());
        assert_eq!(
            &Value(vec![
                PathStep::Name("foo"),
                PathStep::Index("0", 0),
                PathStep::Name("bar")
            ]),
            &instructions[0]
        );
    }

    #[test]
    fn test_mixture() {
        let text = "Hello { name }, how are you?";
        let instructions = compile(text).unwrap();
        assert_eq!(3, instructions.len());
        assert_eq!(&Literal("Hello "), &instructions[0]);
        assert_eq!(&Value(vec![PathStep::Name("name")]), &instructions[1]);
        assert_eq!(&Literal(", how are you?"), &instructions[2]);
    }

    #[test]
    fn test_if_endif() {
        let text = "{{ if foo }}Hello!{{ endif }}";
        let instructions = compile(text).unwrap();
        assert_eq!(2, instructions.len());
        assert_eq!(
            &Branch(vec![PathStep::Name("foo")], true, 2),
            &instructions[0]
        );
        assert_eq!(&Literal("Hello!"), &instructions[1]);
    }

    #[test]
    fn test_if_not_endif() {
        let text = "{{ if not foo }}Hello!{{ endif }}";
        let instructions = compile(text).unwrap();
        assert_eq!(2, instructions.len());
        assert_eq!(
            &Branch(vec![PathStep::Name("foo")], false, 2),
            &instructions[0]
        );
        assert_eq!(&Literal("Hello!"), &instructions[1]);
    }

    #[test]
    fn test_if_else_endif() {
        let text = "{{ if foo }}Hello!{{ else }}Goodbye!{{ endif }}";
        let instructions = compile(text).unwrap();
        assert_eq!(4, instructions.len());
        assert_eq!(
            &Branch(vec![PathStep::Name("foo")], true, 3),
            &instructions[0]
        );
        assert_eq!(&Literal("Hello!"), &instructions[1]);
        assert_eq!(&Goto(4), &instructions[2]);
        assert_eq!(&Literal("Goodbye!"), &instructions[3]);
    }

    #[test]
    fn test_with() {
        let text = "{{ with foo as bar }}Hello!{{ endwith }}";
        let instructions = compile(text).unwrap();
        assert_eq!(3, instructions.len());
        assert_eq!(
            &PushNamedContext(vec![PathStep::Name("foo")], "bar"),
            &instructions[0]
        );
        assert_eq!(&Literal("Hello!"), &instructions[1]);
        assert_eq!(&PopContext, &instructions[2]);
    }

    #[test]
    fn test_foreach() {
        let text = "{{ for foo in bar.baz }}{ foo }{{ endfor }}";
        let instructions = compile(text).unwrap();
        assert_eq!(5, instructions.len());
        assert_eq!(
            &PushIterationContext(vec![PathStep::Name("bar"), PathStep::Name("baz")], "foo"),
            &instructions[0]
        );
        assert_eq!(&Iterate(4), &instructions[1]);
        assert_eq!(&Value(vec![PathStep::Name("foo")]), &instructions[2]);
        assert_eq!(&Goto(1), &instructions[3]);
        assert_eq!(&PopContext, &instructions[4]);
    }

    #[test]
    fn test_strip_whitespace_value() {
        let text = "Hello,     {- name -}   , how are you?";
        let instructions = compile(text).unwrap();
        assert_eq!(3, instructions.len());
        assert_eq!(&Literal("Hello,"), &instructions[0]);
        assert_eq!(&Value(vec![PathStep::Name("name")]), &instructions[1]);
        assert_eq!(&Literal(", how are you?"), &instructions[2]);
    }

    #[test]
    fn test_strip_whitespace_block() {
        let text = "Hello,     {{- if name -}}    {name}    {{- endif -}}   , how are you?";
        let instructions = compile(text).unwrap();
        assert_eq!(6, instructions.len());
        assert_eq!(&Literal("Hello,"), &instructions[0]);
        assert_eq!(
            &Branch(vec![PathStep::Name("name")], true, 5),
            &instructions[1]
        );
        assert_eq!(&Literal(""), &instructions[2]);
        assert_eq!(&Value(vec![PathStep::Name("name")]), &instructions[3]);
        assert_eq!(&Literal(""), &instructions[4]);
        assert_eq!(&Literal(", how are you?"), &instructions[5]);
    }

    #[test]
    fn test_comment() {
        let text = "Hello, {# foo bar baz #} there!";
        let instructions = compile(text).unwrap();
        assert_eq!(2, instructions.len());
        assert_eq!(&Literal("Hello, "), &instructions[0]);
        assert_eq!(&Literal(" there!"), &instructions[1]);
    }

    #[test]
    fn test_strip_whitespace_comment() {
        let text = "Hello, \t\n    {#- foo bar baz -#} \t  there!";
        let instructions = compile(text).unwrap();
        assert_eq!(2, instructions.len());
        assert_eq!(&Literal("Hello,"), &instructions[0]);
        assert_eq!(&Literal("there!"), &instructions[1]);
    }

    #[test]
    fn test_strip_whitespace_followed_by_another_tag() {
        let text = "{value -}{value} Hello";
        let instructions = compile(text).unwrap();
        assert_eq!(3, instructions.len());
        assert_eq!(&Value(vec![PathStep::Name("value")]), &instructions[0]);
        assert_eq!(&Value(vec![PathStep::Name("value")]), &instructions[1]);
        assert_eq!(&Literal(" Hello"), &instructions[2]);
    }

    #[test]
    fn test_call() {
        let text = "{{ call my_macro with foo.bar }}";
        let instructions = compile(text).unwrap();
        assert_eq!(1, instructions.len());
        assert_eq!(
            &Call(
                "my_macro",
                vec![PathStep::Name("foo"), PathStep::Name("bar")]
            ),
            &instructions[0]
        );
    }

    #[test]
    fn test_curly_brace_escaping() {
        let text = "body \\{ \nfont-size: {fontsize} \n}";
        let instructions = compile(text).unwrap();
        assert_eq!(4, instructions.len());
        assert_eq!(&Literal("body "), &instructions[0]);
        assert_eq!(&Literal("{ \nfont-size: "), &instructions[1]);
        assert_eq!(&Value(vec![PathStep::Name("fontsize")]), &instructions[2]);
        assert_eq!(&Literal(" \n}"), &instructions[3]);
    }

    #[test]
    fn test_unclosed_tags() {
        let tags = vec![
            "{",
            "{ foo.bar",
            "{ foo.bar\n }",
            "{{",
            "{{ if foo.bar",
            "{{ if foo.bar \n}}",
            "{#",
            "{# if foo.bar",
            "{# if foo.bar \n#}",
        ];
        for tag in tags {
            compile(tag).unwrap_err();
        }
    }

    #[test]
    fn test_mismatched_blocks() {
        let text = "{{ if foo }}{{ with bar }}{{ endif }} {{ endwith }}";
        compile(text).unwrap_err();
    }

    #[test]
    fn test_disallows_invalid_keywords() {
        let text = "{ @foo }";
        compile(text).unwrap_err();
    }

    #[test]
    fn test_diallows_unknown_block_type() {
        let text = "{{ foobar }}";
        compile(text).unwrap_err();
    }

    #[test]
    fn test_parse_error_line_column_num() {
        let text = "\n\n\n{{ foobar }}";
        let err = compile(text).unwrap_err();
        if let ParseError { line, column, .. } = err {
            assert_eq!(4, line);
            assert_eq!(3, column);
        } else {
            panic!("Should have returned a parse error");
        }
    }

    #[test]
    fn test_parse_error_on_unclosed_if() {
        let text = "{{ if foo }}";
        compile(text).unwrap_err();
    }

    #[test]
    fn test_parse_escaped_open_curly_brace() {
        let text: &str = r"hello \{world}";
        let instructions = compile(text).unwrap();
        assert_eq!(2, instructions.len());
        assert_eq!(&Literal("hello "), &instructions[0]);
        assert_eq!(&Literal("{world}"), &instructions[1]);
    }
}