Read whitespace in tokenstream and reconstruct source content

handlebars/src/main/antlr4/com/github/jknack/handlebars/internal/HbsLexer.g4

@@ -9,6 +9,8 @@ lexer grammar HbsLexer;
 
   boolean whiteSpaceControl;
 
+  public static final int WHITESPACE_CHANNEL = 1;
+
   public HbsLexer(CharStream input, String start, String end) {
     this(input);
     this.start = start;
@@ -374,5 +376,5 @@ RP
   ;
 
 WS
- : [ \t\r\n] -> skip
+ : [ \t\r\n] -> channel(1)
  ;

handlebars/src/main/java/com/github/jknack/handlebars/Template.java

@@ -134,6 +134,20 @@ public List<String> collectReferenceParameters() {
    */
   String text();
 
+  /**
+   * Provide the exact original source text including all whitespace. This method provides lossless
+   * source reconstruction by using whitespace preserved on hidden channels during parsing.
+   *
+   * <p>If token information is not available (for templates created without token streams), this
+   * method falls back to {@link #text()}.
+   *
+   * @return The exact original source text, or reconstructed text if token info unavailable.
+   * @since 4.6.0
+   */
+  default String getSourceText() {
+    return text();
+  }
+
   /**
    * Convert this template to JavaScript template (a.k.a precompiled template). Compilation is done
    * by handlebars.js and a JS Engine (usually Rhino).

handlebars/src/main/java/com/github/jknack/handlebars/internal/BaseTemplate.java

@@ -26,6 +26,7 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.antlr.v4.runtime.CommonTokenStream;
 import org.apache.commons.lang3.StringUtils;
 
 import com.github.jknack.handlebars.Context;
@@ -59,6 +60,18 @@ abstract class BaseTemplate implements Template {
   /** A pre-compiled JavaScript function. */
   private String javaScript;
 
+  /**
+   * Token stream for lossless source reconstruction. Optional - if null, falls back to text()
+   * reconstruction.
+   */
+  protected CommonTokenStream tokenStream;
+
+  /** Start token index in the token stream (inclusive). */
+  protected int startTokenIndex = -1;
+
+  /** End token index in the token stream (inclusive). */
+  protected int endTokenIndex = -1;
+
   /**
    * Creates a new {@link BaseTemplate}.
    *
@@ -194,6 +207,51 @@ public BaseTemplate position(final int line, final int column) {
     return this;
   }
 
+  /**
+   * Set the token stream for lossless source reconstruction.
+   *
+   * @param tokenStream The token stream.
+   * @return This template.
+   */
+  public BaseTemplate tokenStream(final CommonTokenStream tokenStream) {
+    this.tokenStream = tokenStream;
+    return this;
+  }
+
+  /**
+   * Set the token span (start and end indices) for lossless source reconstruction.
+   *
+   * @param startTokenIndex The start token index (inclusive).
+   * @param endTokenIndex The end token index (inclusive).
+   * @return This template.
+   */
+  public BaseTemplate tokenSpan(final int startTokenIndex, final int endTokenIndex) {
+    this.startTokenIndex = startTokenIndex;
+    this.endTokenIndex = endTokenIndex;
+    return this;
+  }
+
+  /**
+   * Get the exact original source text for this template, including all whitespace. This provides
+   * lossless source reconstruction using the whitespace preserved on channel 1.
+   *
+   * <p>If token information is not available (for templates created without token spans), this
+   * falls back to the reconstructed text() method.
+   *
+   * @return The exact original source text, or reconstructed text if token info unavailable.
+   * @since 4.6.0
+   */
+  public String getSourceText() {
+    if (tokenStream != null && startTokenIndex >= 0 && endTokenIndex >= 0) {
+      // Use TokenStream.getText() to get exact original source including all channels
+      // This includes whitespace tokens on channel 1
+      return tokenStream.getText(
+          org.antlr.v4.runtime.misc.Interval.of(startTokenIndex, endTokenIndex));
+    }
+    // Fallback to reconstructed text if token information not available
+    return text();
+  }
+
   @Override
   public <T, S extends TypeSafeTemplate<T>> S as(final Class<S> rootType) {
     notNull(rootType, "The rootType can't be null.");

handlebars/src/main/java/com/github/jknack/handlebars/internal/ForwardingTemplate.java

@@ -111,6 +111,11 @@ public String text() {
     return template.text();
   }
 
+  @Override
+  public String getSourceText() {
+    return template.getSourceText();
+  }
+
   @Override
   public String toJavaScript() {
     return template.toJavaScript();

handlebars/src/main/java/com/github/jknack/handlebars/internal/HbsParserFactory.java

@@ -86,7 +86,7 @@ public Template parse(final TemplateSource source) throws IOException {
 
         /** Build the AST. */
         TemplateBuilder builder =
-            new TemplateBuilder(handlebars, source) {
+            new TemplateBuilder(handlebars, source, (CommonTokenStream) parser.getTokenStream()) {
               @Override
               protected void reportError(
                   final CommonToken offendingToken,

handlebars/src/main/java/com/github/jknack/handlebars/internal/TemplateBuilder.java

@@ -98,6 +98,9 @@ private static class PartialInfo {
   /** The template source. Required. */
   private TemplateSource source;
 
+  /** The token stream for lossless source reconstruction. Optional. */
+  private org.antlr.v4.runtime.CommonTokenStream tokenStream;
+
   /** Flag to track dead spaces and lines. */
   private Boolean hasTag;
 
@@ -124,6 +127,22 @@ private static class PartialInfo {
     this.source = notNull(source, "The template source is required.");
   }
 
+  /**
+   * Creates a new {@link TemplateBuilder} with token stream for lossless source reconstruction.
+   *
+   * @param handlebars A handlebars object. required.
+   * @param source The template source. required.
+   * @param tokenStream The token stream. optional.
+   */
+  TemplateBuilder(
+      final Handlebars handlebars,
+      final TemplateSource source,
+      final org.antlr.v4.runtime.CommonTokenStream tokenStream) {
+    this.handlebars = notNull(handlebars, "The handlebars can't be null.");
+    this.source = notNull(source, "The template source is required.");
+    this.tokenStream = tokenStream;
+  }
+
   @Override
   public Template visit(final ParseTree tree) {
     return (Template) super.visit(tree);
@@ -268,6 +287,17 @@ public Template visitBlock(final BlockContext ctx) {
       Template elsebody = visitBody(elseStmtChainContext.unlessBody);
       elseblock.body(elsebody);
 
+      // Set token span for lossless source reconstruction (else if blocks)
+      if (tokenStream != null
+          && elseStmtChainContext.start != null
+          && elseStmtChainContext.stop != null) {
+        elseblock
+            .tokenStream(tokenStream)
+            .tokenSpan(
+                elseStmtChainContext.start.getTokenIndex(),
+                elseStmtChainContext.stop.getTokenIndex());
+      }
+
       String inverseLabel = elseStmtChainContext.inverseToken.getText();
       if (inverseLabel.startsWith(startDelim)) {
         inverseLabel = inverseLabel.substring(startDelim.length());
@@ -295,6 +325,12 @@ public Template visitBlock(final BlockContext ctx) {
       paramStack.removeLast();
     }
     level -= 1;
+
+    // Set token span for lossless source reconstruction
+    if (tokenStream != null && ctx.start != null && ctx.stop != null) {
+      block.tokenStream(tokenStream).tokenSpan(ctx.start.getTokenIndex(), ctx.stop.getTokenIndex());
+    }
+
     return block;
   }
 
@@ -335,21 +371,29 @@ public Template visitUnless(final UnlessContext ctx) {
     }
     hasTag(true);
     level -= 1;
+
+    // Set token span for lossless source reconstruction
+    if (tokenStream != null && ctx.start != null && ctx.stop != null) {
+      block.tokenStream(tokenStream).tokenSpan(ctx.start.getTokenIndex(), ctx.stop.getTokenIndex());
+    }
+
     return block;
   }
 
   @Override
   public Template visitVar(final VarContext ctx) {
     hasTag(false);
     SexprContext sexpr = ctx.sexpr();
-    return newVar(
+    return newVarWithTokens(
         sexpr.QID().getSymbol(),
         TagType.VAR,
         params(sexpr.param()),
         hash(sexpr.hash()),
         ctx.start.getText(),
         ctx.stop.getText(),
-        ctx.DECORATOR() != null);
+        ctx.DECORATOR() != null,
+        ctx.start,
+        ctx.stop);
   }
 
   @Override
@@ -365,28 +409,32 @@ public Object visitEscape(final EscapeContext ctx) {
   public Template visitTvar(final TvarContext ctx) {
     hasTag(false);
     SexprContext sexpr = ctx.sexpr();
-    return newVar(
+    return newVarWithTokens(
         sexpr.QID().getSymbol(),
         TagType.TRIPLE_VAR,
         params(sexpr.param()),
         hash(sexpr.hash()),
         ctx.start.getText(),
         ctx.stop.getText(),
-        false);
+        false,
+        ctx.start,
+        ctx.stop);
   }
 
   @Override
   public Template visitAmpvar(final AmpvarContext ctx) {
     hasTag(false);
     SexprContext sexpr = ctx.sexpr();
-    return newVar(
+    return newVarWithTokens(
         sexpr.QID().getSymbol(),
         TagType.AMP_VAR,
         params(sexpr.param()),
         hash(sexpr.hash()),
         ctx.start.getText(),
         ctx.stop.getText(),
-        false);
+        false,
+        ctx.start,
+        ctx.stop);
   }
 
   /**
@@ -476,6 +524,37 @@ private Variable newVar(
     return var;
   }
 
+  /**
+   * Build a new {@link Variable} with token span for lossless source reconstruction.
+   *
+   * @param name The var's name.
+   * @param varType The var's type.
+   * @param params The var params.
+   * @param hash The var hash.
+   * @param startDelimiter The current start delimiter.
+   * @param endDelimiter The current end delimiter.
+   * @param decorator True, for var decorators.
+   * @param startToken The start token.
+   * @param stopToken The stop token.
+   * @return A new {@link Variable}.
+   */
+  private Variable newVarWithTokens(
+      final Token name,
+      final TagType varType,
+      final List<Param> params,
+      final Map<String, Param> hash,
+      final String startDelimiter,
+      final String endDelimiter,
+      final boolean decorator,
+      final Token startToken,
+      final Token stopToken) {
+    Variable var = newVar(name, varType, params, hash, startDelimiter, endDelimiter, decorator);
+    if (tokenStream != null && startToken != null && stopToken != null) {
+      var.tokenStream(tokenStream).tokenSpan(startToken.getTokenIndex(), stopToken.getTokenIndex());
+    }
+    return var;
+  }
+
   /**
    * Build a hash.
    *
@@ -577,6 +656,20 @@ public Object visitNumberParam(final HbsParser.NumberParamContext ctx) {
   @Override
   public Template visitTemplate(final TemplateContext ctx) {
     Template template = visitBody(ctx.body());
+
+    // Set token span for lossless source reconstruction
+    if (tokenStream != null && template instanceof BaseTemplate) {
+      BaseTemplate baseTemplate = (BaseTemplate) template;
+      // Use the body's start and stop tokens for the entire template
+      Token startToken = ctx.body().start;
+      Token stopToken = ctx.body().stop;
+      if (startToken != null && stopToken != null) {
+        baseTemplate
+            .tokenStream(tokenStream)
+            .tokenSpan(startToken.getTokenIndex(), stopToken.getTokenIndex());
+      }
+    }
+
     if (!handlebars.infiniteLoops() && template instanceof BaseTemplate) {
       template = infiniteLoop(source, (BaseTemplate) template);
     }
@@ -648,6 +741,16 @@ public Template visitPartial(final PartialContext ctx) {
             .filename(source.filename())
             .position(info.token.getLine(), info.token.getCharPositionInLine());
 
+    // Set token span for lossless source reconstruction
+    if (tokenStream != null
+        && partial instanceof BaseTemplate
+        && ctx.start != null
+        && ctx.stop != null) {
+      ((BaseTemplate) partial)
+          .tokenStream(tokenStream)
+          .tokenSpan(ctx.start.getTokenIndex(), ctx.stop.getTokenIndex());
+    }
+
     return partial;
   }
 
@@ -678,6 +781,16 @@ public Object visitPartialBlock(final PartialBlockContext ctx) {
             .filename(source.filename())
             .position(info.token.getLine(), info.token.getCharPositionInLine());
 
+    // Set token span for lossless source reconstruction
+    if (tokenStream != null
+        && partial instanceof BaseTemplate
+        && ctx.start != null
+        && ctx.stop != null) {
+      ((BaseTemplate) partial)
+          .tokenStream(tokenStream)
+          .tokenSpan(ctx.start.getTokenIndex(), ctx.stop.getTokenIndex());
+    }
+
     return partial;
   }
 
@@ -776,6 +889,11 @@ public Template visitBody(final BodyContext ctx) {
       }
     }
 
+    // Set token span for lossless source reconstruction
+    if (tokenStream != null && ctx.start != null && ctx.stop != null) {
+      list.tokenStream(tokenStream).tokenSpan(ctx.start.getTokenIndex(), ctx.stop.getTokenIndex());
+    }
+
     return list;
   }
 

handlebars/src/main/java/com/github/jknack/handlebars/internal/TemplateList.java

@@ -96,6 +96,15 @@ public String text() {
     return buffer.toString();
   }
 
+  @Override
+  public String getSourceText() {
+    StringBuilder buffer = new StringBuilder();
+    for (Template node : nodes) {
+      buffer.append(node.getSourceText());
+    }
+    return buffer.toString();
+  }
+
   @Override
   public Iterator<Template> iterator() {
     return nodes.iterator();