From 58b5695cd83909a7d08d2639524af7f026b2777a Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Sat, 7 Jun 2025 21:45:12 +0200 Subject: [PATCH 1/3] Add support for hexadecimal and octal escape sequences in string literals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements \xNN (hexadecimal) and \NNN (octal) escape sequences to match Jinja2 compatibility. The \0 sequence is now handled as part of the octal system for consistency. Fixes #803 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- minijinja/src/utils.rs | 55 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/minijinja/src/utils.rs b/minijinja/src/utils.rs index a01e4691..48100398 100644 --- a/minijinja/src/utils.rs +++ b/minijinja/src/utils.rs @@ -277,6 +277,14 @@ impl Unescaper { let val = ok!(self.parse_u16(&mut char_iter)); ok!(self.push_u16(val)); } + 'x' => { + let val = ok!(self.parse_hex_byte(&mut char_iter)); + ok!(self.push_char(val as char)); + } + '0'..='7' => { + let val = ok!(self.parse_octal_byte(d, &mut char_iter)); + ok!(self.push_char(val as char)); + } _ => return Err(ErrorKind::BadEscape.into()), }, } @@ -297,6 +305,33 @@ impl Unescaper { u16::from_str_radix(&hexnum, 16).map_err(|_| ErrorKind::BadEscape.into()) } + fn parse_hex_byte(&self, chars: &mut Chars) -> Result { + let hexnum = chars.chain(repeat('\0')).take(2).collect::(); + u8::from_str_radix(&hexnum, 16).map_err(|_| ErrorKind::BadEscape.into()) + } + + fn parse_octal_byte(&self, first_digit: char, chars: &mut Chars) -> Result { + let mut octal_str = String::new(); + octal_str.push(first_digit); + + // Collect up to 2 more octal digits (0-7) + for _ in 0..2 { + let next_char = chars.as_str().chars().next(); + if let Some(c) = next_char { + if c >= '0' && c <= '7' { + octal_str.push(c); + chars.next(); // consume the character + } else { + break; + } + } else { + break; + } + } + + u8::from_str_radix(&octal_str, 8).map_err(|_| ErrorKind::BadEscape.into()) + } + fn push_u16(&mut self, c: u16) -> Result<(), Error> { match (self.pending_surrogate, (0xD800..=0xDFFF).contains(&c)) { (0, false) => match decode_utf16(once(c)).next() { @@ -450,6 +485,26 @@ mod tests { assert_eq!(unescape(r"\t\b\f\r\n\\\/").unwrap(), "\t\x08\x0c\r\n\\/"); assert_eq!(unescape("foobarbaz").unwrap(), "foobarbaz"); assert_eq!(unescape(r"\ud83d\udca9").unwrap(), "💩"); + + // Test new escape sequences + assert_eq!(unescape(r"\0").unwrap(), "\0"); + assert_eq!(unescape(r"foo\0bar").unwrap(), "foo\0bar"); + assert_eq!(unescape(r"\x00").unwrap(), "\0"); + assert_eq!(unescape(r"\x42").unwrap(), "B"); + assert_eq!(unescape(r"\xab").unwrap(), "\u{ab}"); + assert_eq!(unescape(r"foo\x42bar").unwrap(), "fooBbar"); + assert_eq!(unescape(r"\x0a").unwrap(), "\n"); + assert_eq!(unescape(r"\x0d").unwrap(), "\r"); + + // Test octal escape sequences + assert_eq!(unescape(r"\0").unwrap(), "\0"); // octal 0 = null + assert_eq!(unescape(r"\1").unwrap(), "\x01"); // octal 1 = SOH + assert_eq!(unescape(r"\12").unwrap(), "\n"); // octal 12 = 10 decimal = LF + assert_eq!(unescape(r"\123").unwrap(), "S"); // octal 123 = 83 decimal = 'S' + assert_eq!(unescape(r"\141").unwrap(), "a"); // octal 141 = 97 decimal = 'a' + assert_eq!(unescape(r"\177").unwrap(), "\x7f"); // octal 177 = 127 decimal = DEL + assert_eq!(unescape(r"foo\123bar").unwrap(), "fooSbar"); // 'S' in the middle + assert_eq!(unescape(r"\101\102\103").unwrap(), "ABC"); // octal for A, B, C } #[test] From d0caa10040f6b2963482a2e5a13c7eed9df59591 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Sat, 7 Jun 2025 21:50:33 +0200 Subject: [PATCH 2/3] Update utils.rs Make clippy happy --- minijinja/src/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minijinja/src/utils.rs b/minijinja/src/utils.rs index 48100398..5013879c 100644 --- a/minijinja/src/utils.rs +++ b/minijinja/src/utils.rs @@ -318,7 +318,7 @@ impl Unescaper { for _ in 0..2 { let next_char = chars.as_str().chars().next(); if let Some(c) = next_char { - if c >= '0' && c <= '7' { + if ('0'..='7').contains(&c) { octal_str.push(c); chars.next(); // consume the character } else { From ca47f3b55299a548eee347dd6f2b25663ee4c357 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Sat, 7 Jun 2025 21:59:38 +0200 Subject: [PATCH 3/3] Add Python-compatible rendering mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an optional pycompat_rendering flag to Environment that enables rendering values in a format compatible with Python Jinja2: - `true`/`false` render as `True`/`False` - `none` renders as `None` - Strings use Python-style escaping and quoting This addresses compatibility issues for SQL templating and other use cases where precise output format matters. The feature is disabled by default to maintain backward compatibility. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- Cargo.lock | 7 + examples/pycompat-rendering/Cargo.toml | 7 + examples/pycompat-rendering/README.md | 37 +++++ examples/pycompat-rendering/src/main.rs | 72 ++++++++++ minijinja/src/environment.rs | 21 +++ minijinja/src/value/mod.rs | 104 +++++++++++++- minijinja/src/vm/mod.rs | 1 + minijinja/tests/test_pycompat_rendering.rs | 158 +++++++++++++++++++++ 8 files changed, 401 insertions(+), 6 deletions(-) create mode 100644 examples/pycompat-rendering/Cargo.toml create mode 100644 examples/pycompat-rendering/README.md create mode 100644 examples/pycompat-rendering/src/main.rs create mode 100644 minijinja/tests/test_pycompat_rendering.rs diff --git a/Cargo.lock b/Cargo.lock index 514b2d07..a5b868cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2448,6 +2448,13 @@ dependencies = [ "cc", ] +[[package]] +name = "pycompat-rendering" +version = "0.1.0" +dependencies = [ + "minijinja", +] + [[package]] name = "pyo3" version = "0.23.5" diff --git a/examples/pycompat-rendering/Cargo.toml b/examples/pycompat-rendering/Cargo.toml new file mode 100644 index 00000000..7c972412 --- /dev/null +++ b/examples/pycompat-rendering/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "pycompat-rendering" +version = "0.1.0" +edition = "2021" + +[dependencies] +minijinja = { path = "../../minijinja" } \ No newline at end of file diff --git a/examples/pycompat-rendering/README.md b/examples/pycompat-rendering/README.md new file mode 100644 index 00000000..48aa7977 --- /dev/null +++ b/examples/pycompat-rendering/README.md @@ -0,0 +1,37 @@ +# Python-Compatible Rendering Example + +This example demonstrates MiniJinja's Python-compatible rendering mode. + +## Background + +By default, MiniJinja renders values in its own style: +- Booleans: `true`/`false` +- None: `none` +- Strings: Rust-style escaping and quoting + +When Python compatibility is needed (e.g., for SQL templating where precise output matters), you can enable PyCompat mode to match Python Jinja2's output: +- Booleans: `True`/`False` +- None: `None` +- Strings: Python-style escaping and quoting + +## Usage + +```rust +use minijinja::Environment; + +let mut env = Environment::new(); + +// Enable Python-compatible rendering +env.set_pycompat_rendering(true); + +// Now templates will render values like Python Jinja2 +let tmpl = env.template_from_str("{{ [true, false, none, 'hello'] }}")?; +let result = tmpl.render(minijinja::context!{})?; +// Output: [True, False, None, 'hello'] +``` + +## Running this Example + +```bash +cargo run --example pycompat-rendering +``` \ No newline at end of file diff --git a/examples/pycompat-rendering/src/main.rs b/examples/pycompat-rendering/src/main.rs new file mode 100644 index 00000000..3b84ce7b --- /dev/null +++ b/examples/pycompat-rendering/src/main.rs @@ -0,0 +1,72 @@ +use minijinja::{context, Environment}; + +fn main() -> Result<(), Box> { + println!("=== MiniJinja Rendering Comparison ==="); + + // Create environments for both modes to avoid borrowing issues + let mut env_default = Environment::new(); + let mut env_pycompat = Environment::new(); + + // Add the same template to both environments + let template_str = r#"{{ [true, false, none, 'foo', "bar'baz", '\x13'] }}"#; + env_default.add_template("demo", template_str)?; + env_pycompat.add_template("demo", template_str)?; + + // Configure rendering modes + env_default.set_pycompat_rendering(false); + env_pycompat.set_pycompat_rendering(true); + + // Test rendering + let result_default = env_default.get_template("demo")?.render(context! {})?; + let result_pycompat = env_pycompat.get_template("demo")?.render(context! {})?; + + println!("Default rendering: {}", result_default); + println!("PyCompat rendering: {}", result_pycompat); + + println!("\n=== Individual Value Comparison ==="); + + // Test individual values + env_default.add_template("bool_true", "{{ true }}")?; + env_default.add_template("bool_false", "{{ false }}")?; + env_default.add_template("none_val", "{{ none }}")?; + + env_pycompat.add_template("bool_true", "{{ true }}")?; + env_pycompat.add_template("bool_false", "{{ false }}")?; + env_pycompat.add_template("none_val", "{{ none }}")?; + + println!("Default mode:"); + println!( + " true -> {}", + env_default.get_template("bool_true")?.render(context! {})? + ); + println!( + " false -> {}", + env_default + .get_template("bool_false")? + .render(context! {})? + ); + println!( + " none -> {}", + env_default.get_template("none_val")?.render(context! {})? + ); + + println!("PyCompat mode:"); + println!( + " true -> {}", + env_pycompat + .get_template("bool_true")? + .render(context! {})? + ); + println!( + " false -> {}", + env_pycompat + .get_template("bool_false")? + .render(context! {})? + ); + println!( + " none -> {}", + env_pycompat.get_template("none_val")?.render(context! {})? + ); + + Ok(()) +} diff --git a/minijinja/src/environment.rs b/minijinja/src/environment.rs index f5b997b5..b587ad78 100644 --- a/minijinja/src/environment.rs +++ b/minijinja/src/environment.rs @@ -59,6 +59,7 @@ pub struct Environment<'source> { #[cfg(feature = "fuel")] fuel: Option, recursion_limit: usize, + pycompat_rendering: bool, } impl Default for Environment<'_> { @@ -111,6 +112,7 @@ impl<'source> Environment<'source> { #[cfg(feature = "fuel")] fuel: None, recursion_limit: MAX_RECURSION, + pycompat_rendering: false, } } @@ -133,6 +135,7 @@ impl<'source> Environment<'source> { #[cfg(feature = "fuel")] fuel: None, recursion_limit: MAX_RECURSION, + pycompat_rendering: false, } } @@ -576,6 +579,24 @@ impl<'source> Environment<'source> { self.debug } + /// Enables or disables Python-compatible rendering mode. + /// + /// When enabled, certain value types will be rendered to match Python Jinja2's output: + /// - `true`/`false` will be rendered as `True`/`False` + /// - `none` will be rendered as `None` + /// - String escaping and quoting will match Python's style + /// + /// This is useful when you need template output that's compatible with Python Jinja2. + /// By default this is disabled. + pub fn set_pycompat_rendering(&mut self, enabled: bool) { + self.pycompat_rendering = enabled; + } + + /// Returns whether Python-compatible rendering mode is enabled. + pub fn pycompat_rendering(&self) -> bool { + self.pycompat_rendering + } + /// Sets the optional fuel of the engine. /// /// When MiniJinja is compiled with the `fuel` feature then every diff --git a/minijinja/src/value/mod.rs b/minijinja/src/value/mod.rs index 72a66908..9de77344 100644 --- a/minijinja/src/value/mod.rs +++ b/minijinja/src/value/mod.rs @@ -259,6 +259,7 @@ pub(crate) fn value_map_with_capacity(capacity: usize) -> ValueMap { thread_local! { static INTERNAL_SERIALIZATION: Cell = const { Cell::new(false) }; + static PYCOMPAT_RENDERING: Cell = const { Cell::new(false) }; // This should be an AtomicU64 but sadly 32bit targets do not necessarily have // AtomicU64 available. @@ -286,6 +287,14 @@ pub fn serializing_for_value() -> bool { INTERNAL_SERIALIZATION.with(|flag| flag.get()) } +/// Function that returns true when Python-compatible rendering is enabled. +/// +/// When this returns `true`, values should be rendered in a way that's compatible +/// with Python Jinja2, such as `True`/`False` instead of `true`/`false`. +pub(crate) fn pycompat_rendering() -> bool { + PYCOMPAT_RENDERING.with(|flag| flag.get()) +} + fn mark_internal_serialization() -> impl Drop { let old = INTERNAL_SERIALIZATION.with(|flag| { let old = flag.get(); @@ -299,6 +308,17 @@ fn mark_internal_serialization() -> impl Drop { }) } +pub(crate) fn mark_pycompat_rendering(enabled: bool) -> impl Drop { + let old = PYCOMPAT_RENDERING.with(|flag| { + let old = flag.get(); + flag.set(enabled); + old + }); + OnDrop::new(move || { + PYCOMPAT_RENDERING.with(|flag| flag.set(old)); + }) +} + /// Describes the kind of value. #[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] #[non_exhaustive] @@ -434,20 +454,80 @@ pub(crate) enum ValueRepr { Object(DynObject), } +fn python_string_debug_fmt(s: &str, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // Python prefers single quotes unless the string contains single quotes + let contains_single = s.contains('\''); + let contains_double = s.contains('"'); + + let quote_char = if contains_single && !contains_double { + '"' + } else { + '\'' + }; + + write!(f, "{}", quote_char)?; + for ch in s.chars() { + match ch { + '\'' if quote_char == '\'' => write!(f, "\\'")?, + '"' if quote_char == '"' => write!(f, "\\\"")?, + '\\' => write!(f, "\\\\")?, + '\n' => write!(f, "\\n")?, + '\r' => write!(f, "\\r")?, + '\t' => write!(f, "\\t")?, + c if c.is_control() => { + let code = c as u32; + if code <= 0xFF { + write!(f, "\\x{:02x}", code)?; + } else if code <= 0xFFFF { + write!(f, "\\u{:04x}", code)?; + } else { + write!(f, "\\U{:08x}", code)?; + } + } + c => write!(f, "{}", c)?, + } + } + write!(f, "{}", quote_char) +} + impl fmt::Debug for ValueRepr { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match *self { ValueRepr::Undefined(_) => f.write_str("undefined"), - ValueRepr::Bool(ref val) => fmt::Debug::fmt(val, f), + ValueRepr::Bool(ref val) => { + if pycompat_rendering() { + write!(f, "{}", if *val { "True" } else { "False" }) + } else { + fmt::Debug::fmt(val, f) + } + } ValueRepr::U64(ref val) => fmt::Debug::fmt(val, f), ValueRepr::I64(ref val) => fmt::Debug::fmt(val, f), ValueRepr::F64(ref val) => fmt::Debug::fmt(val, f), - ValueRepr::None => f.write_str("none"), + ValueRepr::None => { + if pycompat_rendering() { + f.write_str("None") + } else { + f.write_str("none") + } + } ValueRepr::Invalid(ref val) => write!(f, "", val), ValueRepr::U128(val) => fmt::Debug::fmt(&{ val.0 }, f), ValueRepr::I128(val) => fmt::Debug::fmt(&{ val.0 }, f), - ValueRepr::String(ref val, _) => fmt::Debug::fmt(val, f), - ValueRepr::SmallStr(ref val) => fmt::Debug::fmt(val.as_str(), f), + ValueRepr::String(ref val, _) => { + if pycompat_rendering() { + python_string_debug_fmt(val, f) + } else { + fmt::Debug::fmt(val, f) + } + } + ValueRepr::SmallStr(ref val) => { + if pycompat_rendering() { + python_string_debug_fmt(val.as_str(), f) + } else { + fmt::Debug::fmt(val.as_str(), f) + } + } ValueRepr::Bytes(ref val) => { write!(f, "b'")?; for &b in val.iter() { @@ -661,7 +741,13 @@ impl fmt::Display for Value { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self.0 { ValueRepr::Undefined(_) => Ok(()), - ValueRepr::Bool(val) => val.fmt(f), + ValueRepr::Bool(val) => { + if pycompat_rendering() { + write!(f, "{}", if val { "True" } else { "False" }) + } else { + val.fmt(f) + } + } ValueRepr::U64(val) => val.fmt(f), ValueRepr::I64(val) => val.fmt(f), ValueRepr::F64(val) => { @@ -677,7 +763,13 @@ impl fmt::Display for Value { write!(f, "{num}") } } - ValueRepr::None => f.write_str("none"), + ValueRepr::None => { + if pycompat_rendering() { + f.write_str("None") + } else { + f.write_str("none") + } + } ValueRepr::Invalid(ref val) => write!(f, "", val), ValueRepr::I128(val) => write!(f, "{}", { val.0 }), ValueRepr::String(ref val, _) => write!(f, "{val}"), diff --git a/minijinja/src/vm/mod.rs b/minijinja/src/vm/mod.rs index a53d0e1d..7831a23e 100644 --- a/minijinja/src/vm/mod.rs +++ b/minijinja/src/vm/mod.rs @@ -91,6 +91,7 @@ impl<'env> Vm<'env> { out: &mut Output, auto_escape: AutoEscape, ) -> Result<(Option, State<'template, 'env>), Error> { + let _pycompat_guard = crate::value::mark_pycompat_rendering(self.env.pycompat_rendering()); let mut state = State::new( Context::new_with_frame(self.env, ok!(Frame::new_checked(root))), auto_escape, diff --git a/minijinja/tests/test_pycompat_rendering.rs b/minijinja/tests/test_pycompat_rendering.rs new file mode 100644 index 00000000..41bc0dd4 --- /dev/null +++ b/minijinja/tests/test_pycompat_rendering.rs @@ -0,0 +1,158 @@ +use minijinja::{context, Environment}; + +#[test] +fn test_pycompat_rendering_boolean_values() { + let mut env = Environment::new(); + + env.add_template("bool_true", "{{ true }}").unwrap(); + env.add_template("bool_false", "{{ false }}").unwrap(); + + // Test default rendering + env.set_pycompat_rendering(false); + let true_default = env + .get_template("bool_true") + .unwrap() + .render(context! {}) + .unwrap(); + let false_default = env + .get_template("bool_false") + .unwrap() + .render(context! {}) + .unwrap(); + + assert_eq!(true_default, "true"); + assert_eq!(false_default, "false"); + + // Test pycompat rendering + env.set_pycompat_rendering(true); + let true_pycompat = env + .get_template("bool_true") + .unwrap() + .render(context! {}) + .unwrap(); + let false_pycompat = env + .get_template("bool_false") + .unwrap() + .render(context! {}) + .unwrap(); + + assert_eq!(true_pycompat, "True"); + assert_eq!(false_pycompat, "False"); +} + +#[test] +fn test_pycompat_rendering_none_value() { + let mut env = Environment::new(); + + env.add_template("none_val", "{{ none }}").unwrap(); + + // Test default rendering + env.set_pycompat_rendering(false); + let none_default = env + .get_template("none_val") + .unwrap() + .render(context! {}) + .unwrap(); + + assert_eq!(none_default, "none"); + + // Test pycompat rendering + env.set_pycompat_rendering(true); + let none_pycompat = env + .get_template("none_val") + .unwrap() + .render(context! {}) + .unwrap(); + + assert_eq!(none_pycompat, "None"); +} + +#[test] +fn test_pycompat_rendering_array() { + let mut env = Environment::new(); + + env.add_template("array", "{{ [true, false, none] }}") + .unwrap(); + + // Test default rendering + env.set_pycompat_rendering(false); + let array_default = env + .get_template("array") + .unwrap() + .render(context! {}) + .unwrap(); + + assert_eq!(array_default, "[true, false, none]"); + + // Test pycompat rendering + env.set_pycompat_rendering(true); + let array_pycompat = env + .get_template("array") + .unwrap() + .render(context! {}) + .unwrap(); + + assert_eq!(array_pycompat, "[True, False, None]"); +} + +#[test] +fn test_pycompat_rendering_complex_case() { + let mut env = Environment::new(); + + // Test the specific case from the issue: {{ [true, false, none, 'foo', "bar'baz", '\x13'] }} + env.add_template("complex", "{{ [true, false, none, 'foo'] }}") + .unwrap(); + + // Test default rendering + env.set_pycompat_rendering(false); + let _complex_default = env + .get_template("complex") + .unwrap() + .render(context! {}) + .unwrap(); + + // Test pycompat rendering + env.set_pycompat_rendering(true); + let complex_pycompat = env + .get_template("complex") + .unwrap() + .render(context! {}) + .unwrap(); + + // Check that pycompat has Python-style values + assert!(complex_pycompat.contains("True")); + assert!(complex_pycompat.contains("False")); + assert!(complex_pycompat.contains("None")); + assert!(complex_pycompat.contains("'foo'")); +} + +#[test] +fn test_pycompat_rendering_exact_github_case() { + let mut env = Environment::new(); + + // Test the exact case from the GitHub issue + env.add_template( + "github_case", + r#"{{ [true, false, none, 'foo', "bar'baz", '\x13'] }}"#, + ) + .unwrap(); + + // Test pycompat rendering + env.set_pycompat_rendering(true); + let result = env + .get_template("github_case") + .unwrap() + .render(context! {}) + .unwrap(); + + println!("GitHub case result: {}", result); + + // Check that pycompat has Python-style values + assert!(result.contains("True")); + assert!(result.contains("False")); + assert!(result.contains("None")); + assert!(result.contains("'foo'")); + // Check for proper string quoting and escaping + assert!(result.contains("\"bar'baz\"") || result.contains("'bar\\'baz'")); + assert!(result.contains("'\\x13'")); +}