diff --git a/handlebars/src/main/antlr4/com/github/jknack/handlebars/internal/HbsLexer.g4 b/handlebars/src/main/antlr4/com/github/jknack/handlebars/internal/HbsLexer.g4 index bc6727a7..e09c5937 100644 --- a/handlebars/src/main/antlr4/com/github/jknack/handlebars/internal/HbsLexer.g4 +++ b/handlebars/src/main/antlr4/com/github/jknack/handlebars/internal/HbsLexer.g4 @@ -9,6 +9,8 @@ lexer grammar HbsLexer; boolean whiteSpaceControl; + public static final int WHITESPACE_CHANNEL = 1; + public HbsLexer(CharStream input, String start, String end) { this(input); this.start = start; @@ -374,5 +376,5 @@ RP ; WS - : [ \t\r\n] -> skip + : [ \t\r\n] -> channel(1) ; diff --git a/handlebars/src/main/java/com/github/jknack/handlebars/Template.java b/handlebars/src/main/java/com/github/jknack/handlebars/Template.java index dd87aa1e..96337c87 100644 --- a/handlebars/src/main/java/com/github/jknack/handlebars/Template.java +++ b/handlebars/src/main/java/com/github/jknack/handlebars/Template.java @@ -134,6 +134,20 @@ public List collectReferenceParameters() { */ String text(); + /** + * Provide the exact original source text including all whitespace. This method provides lossless + * source reconstruction by using whitespace preserved on hidden channels during parsing. + * + *

If token information is not available (for templates created without token streams), this + * method falls back to {@link #text()}. + * + * @return The exact original source text, or reconstructed text if token info unavailable. + * @since 4.6.0 + */ + default String getSourceText() { + return text(); + } + /** * Convert this template to JavaScript template (a.k.a precompiled template). Compilation is done * by handlebars.js and a JS Engine (usually Rhino). diff --git a/handlebars/src/main/java/com/github/jknack/handlebars/internal/BaseTemplate.java b/handlebars/src/main/java/com/github/jknack/handlebars/internal/BaseTemplate.java index 2909e066..490f3b2f 100644 --- a/handlebars/src/main/java/com/github/jknack/handlebars/internal/BaseTemplate.java +++ b/handlebars/src/main/java/com/github/jknack/handlebars/internal/BaseTemplate.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.Set; +import org.antlr.v4.runtime.CommonTokenStream; import org.apache.commons.lang3.StringUtils; import com.github.jknack.handlebars.Context; @@ -59,6 +60,18 @@ abstract class BaseTemplate implements Template { /** A pre-compiled JavaScript function. */ private String javaScript; + /** + * Token stream for lossless source reconstruction. Optional - if null, falls back to text() + * reconstruction. + */ + protected CommonTokenStream tokenStream; + + /** Start token index in the token stream (inclusive). */ + protected int startTokenIndex = -1; + + /** End token index in the token stream (inclusive). */ + protected int endTokenIndex = -1; + /** * Creates a new {@link BaseTemplate}. * @@ -194,6 +207,51 @@ public BaseTemplate position(final int line, final int column) { return this; } + /** + * Set the token stream for lossless source reconstruction. + * + * @param tokenStream The token stream. + * @return This template. + */ + public BaseTemplate tokenStream(final CommonTokenStream tokenStream) { + this.tokenStream = tokenStream; + return this; + } + + /** + * Set the token span (start and end indices) for lossless source reconstruction. + * + * @param startTokenIndex The start token index (inclusive). + * @param endTokenIndex The end token index (inclusive). + * @return This template. + */ + public BaseTemplate tokenSpan(final int startTokenIndex, final int endTokenIndex) { + this.startTokenIndex = startTokenIndex; + this.endTokenIndex = endTokenIndex; + return this; + } + + /** + * Get the exact original source text for this template, including all whitespace. This provides + * lossless source reconstruction using the whitespace preserved on channel 1. + * + *

If token information is not available (for templates created without token spans), this + * falls back to the reconstructed text() method. + * + * @return The exact original source text, or reconstructed text if token info unavailable. + * @since 4.6.0 + */ + public String getSourceText() { + if (tokenStream != null && startTokenIndex >= 0 && endTokenIndex >= 0) { + // Use TokenStream.getText() to get exact original source including all channels + // This includes whitespace tokens on channel 1 + return tokenStream.getText( + org.antlr.v4.runtime.misc.Interval.of(startTokenIndex, endTokenIndex)); + } + // Fallback to reconstructed text if token information not available + return text(); + } + @Override public > S as(final Class rootType) { notNull(rootType, "The rootType can't be null."); diff --git a/handlebars/src/main/java/com/github/jknack/handlebars/internal/ForwardingTemplate.java b/handlebars/src/main/java/com/github/jknack/handlebars/internal/ForwardingTemplate.java index d2ed86b3..281474e9 100644 --- a/handlebars/src/main/java/com/github/jknack/handlebars/internal/ForwardingTemplate.java +++ b/handlebars/src/main/java/com/github/jknack/handlebars/internal/ForwardingTemplate.java @@ -111,6 +111,11 @@ public String text() { return template.text(); } + @Override + public String getSourceText() { + return template.getSourceText(); + } + @Override public String toJavaScript() { return template.toJavaScript(); diff --git a/handlebars/src/main/java/com/github/jknack/handlebars/internal/HbsParserFactory.java b/handlebars/src/main/java/com/github/jknack/handlebars/internal/HbsParserFactory.java index ce32afb5..4a59d3aa 100644 --- a/handlebars/src/main/java/com/github/jknack/handlebars/internal/HbsParserFactory.java +++ b/handlebars/src/main/java/com/github/jknack/handlebars/internal/HbsParserFactory.java @@ -86,7 +86,7 @@ public Template parse(final TemplateSource source) throws IOException { /** Build the AST. */ TemplateBuilder builder = - new TemplateBuilder(handlebars, source) { + new TemplateBuilder(handlebars, source, (CommonTokenStream) parser.getTokenStream()) { @Override protected void reportError( final CommonToken offendingToken, diff --git a/handlebars/src/main/java/com/github/jknack/handlebars/internal/TemplateBuilder.java b/handlebars/src/main/java/com/github/jknack/handlebars/internal/TemplateBuilder.java index 291ded55..d04bf530 100644 --- a/handlebars/src/main/java/com/github/jknack/handlebars/internal/TemplateBuilder.java +++ b/handlebars/src/main/java/com/github/jknack/handlebars/internal/TemplateBuilder.java @@ -98,6 +98,9 @@ private static class PartialInfo { /** The template source. Required. */ private TemplateSource source; + /** The token stream for lossless source reconstruction. Optional. */ + private org.antlr.v4.runtime.CommonTokenStream tokenStream; + /** Flag to track dead spaces and lines. */ private Boolean hasTag; @@ -124,6 +127,22 @@ private static class PartialInfo { this.source = notNull(source, "The template source is required."); } + /** + * Creates a new {@link TemplateBuilder} with token stream for lossless source reconstruction. + * + * @param handlebars A handlebars object. required. + * @param source The template source. required. + * @param tokenStream The token stream. optional. + */ + TemplateBuilder( + final Handlebars handlebars, + final TemplateSource source, + final org.antlr.v4.runtime.CommonTokenStream tokenStream) { + this.handlebars = notNull(handlebars, "The handlebars can't be null."); + this.source = notNull(source, "The template source is required."); + this.tokenStream = tokenStream; + } + @Override public Template visit(final ParseTree tree) { return (Template) super.visit(tree); @@ -268,6 +287,17 @@ public Template visitBlock(final BlockContext ctx) { Template elsebody = visitBody(elseStmtChainContext.unlessBody); elseblock.body(elsebody); + // Set token span for lossless source reconstruction (else if blocks) + if (tokenStream != null + && elseStmtChainContext.start != null + && elseStmtChainContext.stop != null) { + elseblock + .tokenStream(tokenStream) + .tokenSpan( + elseStmtChainContext.start.getTokenIndex(), + elseStmtChainContext.stop.getTokenIndex()); + } + String inverseLabel = elseStmtChainContext.inverseToken.getText(); if (inverseLabel.startsWith(startDelim)) { inverseLabel = inverseLabel.substring(startDelim.length()); @@ -295,6 +325,12 @@ public Template visitBlock(final BlockContext ctx) { paramStack.removeLast(); } level -= 1; + + // Set token span for lossless source reconstruction + if (tokenStream != null && ctx.start != null && ctx.stop != null) { + block.tokenStream(tokenStream).tokenSpan(ctx.start.getTokenIndex(), ctx.stop.getTokenIndex()); + } + return block; } @@ -335,6 +371,12 @@ public Template visitUnless(final UnlessContext ctx) { } hasTag(true); level -= 1; + + // Set token span for lossless source reconstruction + if (tokenStream != null && ctx.start != null && ctx.stop != null) { + block.tokenStream(tokenStream).tokenSpan(ctx.start.getTokenIndex(), ctx.stop.getTokenIndex()); + } + return block; } @@ -342,14 +384,16 @@ public Template visitUnless(final UnlessContext ctx) { public Template visitVar(final VarContext ctx) { hasTag(false); SexprContext sexpr = ctx.sexpr(); - return newVar( + return newVarWithTokens( sexpr.QID().getSymbol(), TagType.VAR, params(sexpr.param()), hash(sexpr.hash()), ctx.start.getText(), ctx.stop.getText(), - ctx.DECORATOR() != null); + ctx.DECORATOR() != null, + ctx.start, + ctx.stop); } @Override @@ -365,28 +409,32 @@ public Object visitEscape(final EscapeContext ctx) { public Template visitTvar(final TvarContext ctx) { hasTag(false); SexprContext sexpr = ctx.sexpr(); - return newVar( + return newVarWithTokens( sexpr.QID().getSymbol(), TagType.TRIPLE_VAR, params(sexpr.param()), hash(sexpr.hash()), ctx.start.getText(), ctx.stop.getText(), - false); + false, + ctx.start, + ctx.stop); } @Override public Template visitAmpvar(final AmpvarContext ctx) { hasTag(false); SexprContext sexpr = ctx.sexpr(); - return newVar( + return newVarWithTokens( sexpr.QID().getSymbol(), TagType.AMP_VAR, params(sexpr.param()), hash(sexpr.hash()), ctx.start.getText(), ctx.stop.getText(), - false); + false, + ctx.start, + ctx.stop); } /** @@ -476,6 +524,37 @@ private Variable newVar( return var; } + /** + * Build a new {@link Variable} with token span for lossless source reconstruction. + * + * @param name The var's name. + * @param varType The var's type. + * @param params The var params. + * @param hash The var hash. + * @param startDelimiter The current start delimiter. + * @param endDelimiter The current end delimiter. + * @param decorator True, for var decorators. + * @param startToken The start token. + * @param stopToken The stop token. + * @return A new {@link Variable}. + */ + private Variable newVarWithTokens( + final Token name, + final TagType varType, + final List params, + final Map hash, + final String startDelimiter, + final String endDelimiter, + final boolean decorator, + final Token startToken, + final Token stopToken) { + Variable var = newVar(name, varType, params, hash, startDelimiter, endDelimiter, decorator); + if (tokenStream != null && startToken != null && stopToken != null) { + var.tokenStream(tokenStream).tokenSpan(startToken.getTokenIndex(), stopToken.getTokenIndex()); + } + return var; + } + /** * Build a hash. * @@ -577,6 +656,20 @@ public Object visitNumberParam(final HbsParser.NumberParamContext ctx) { @Override public Template visitTemplate(final TemplateContext ctx) { Template template = visitBody(ctx.body()); + + // Set token span for lossless source reconstruction + if (tokenStream != null && template instanceof BaseTemplate) { + BaseTemplate baseTemplate = (BaseTemplate) template; + // Use the body's start and stop tokens for the entire template + Token startToken = ctx.body().start; + Token stopToken = ctx.body().stop; + if (startToken != null && stopToken != null) { + baseTemplate + .tokenStream(tokenStream) + .tokenSpan(startToken.getTokenIndex(), stopToken.getTokenIndex()); + } + } + if (!handlebars.infiniteLoops() && template instanceof BaseTemplate) { template = infiniteLoop(source, (BaseTemplate) template); } @@ -648,6 +741,16 @@ public Template visitPartial(final PartialContext ctx) { .filename(source.filename()) .position(info.token.getLine(), info.token.getCharPositionInLine()); + // Set token span for lossless source reconstruction + if (tokenStream != null + && partial instanceof BaseTemplate + && ctx.start != null + && ctx.stop != null) { + ((BaseTemplate) partial) + .tokenStream(tokenStream) + .tokenSpan(ctx.start.getTokenIndex(), ctx.stop.getTokenIndex()); + } + return partial; } @@ -678,6 +781,16 @@ public Object visitPartialBlock(final PartialBlockContext ctx) { .filename(source.filename()) .position(info.token.getLine(), info.token.getCharPositionInLine()); + // Set token span for lossless source reconstruction + if (tokenStream != null + && partial instanceof BaseTemplate + && ctx.start != null + && ctx.stop != null) { + ((BaseTemplate) partial) + .tokenStream(tokenStream) + .tokenSpan(ctx.start.getTokenIndex(), ctx.stop.getTokenIndex()); + } + return partial; } @@ -776,6 +889,11 @@ public Template visitBody(final BodyContext ctx) { } } + // Set token span for lossless source reconstruction + if (tokenStream != null && ctx.start != null && ctx.stop != null) { + list.tokenStream(tokenStream).tokenSpan(ctx.start.getTokenIndex(), ctx.stop.getTokenIndex()); + } + return list; } diff --git a/handlebars/src/main/java/com/github/jknack/handlebars/internal/TemplateList.java b/handlebars/src/main/java/com/github/jknack/handlebars/internal/TemplateList.java index 6c106e3d..a334f71d 100644 --- a/handlebars/src/main/java/com/github/jknack/handlebars/internal/TemplateList.java +++ b/handlebars/src/main/java/com/github/jknack/handlebars/internal/TemplateList.java @@ -96,6 +96,15 @@ public String text() { return buffer.toString(); } + @Override + public String getSourceText() { + StringBuilder buffer = new StringBuilder(); + for (Template node : nodes) { + buffer.append(node.getSourceText()); + } + return buffer.toString(); + } + @Override public Iterator