- The HTML cleaner parses the input as HTML and then runs it through a safe-list, so the output HTML can only contain + The HTML cleaner parses the input as HTML and then runs it through a safelist, so the output HTML can only contain HTML that is allowed by the safelist.
It is assumed that the input HTML is a body fragment; the clean methods only pull from the source's body, and the - canned safe-lists only allow body contained tags. + canned safelists only allow body-contained tags.
Rather than interacting directly with a Cleaner object, generally see the {@code clean} methods in {@link org.jsoup.Jsoup}.
++ A Cleaner may be reused across multiple documents and shared across concurrent threads once its {@link Safelist} has + been configured. The cleaner uses the supplied safelist directly, so later safelist changes affect later cleaning + calls. If you need a variant of an existing configuration, use {@link Safelist#Safelist(Safelist)} to make a copy. +
*/ public class Cleaner { private final Safelist safelist; diff --git a/src/main/java/org/jsoup/safety/Safelist.java b/src/main/java/org/jsoup/safety/Safelist.java index cf76c08d3b..e24ab860ab 100644 --- a/src/main/java/org/jsoup/safety/Safelist.java +++ b/src/main/java/org/jsoup/safety/Safelist.java @@ -22,7 +22,7 @@ Thank you to Ryan Grove (wonko.com) for the Ruby HTML cleaner http://github.com/ /** - Safe-lists define what HTML (elements and attributes) to allow through the cleaner. Everything else is removed. + Safelists define what HTML (elements and attributes) to allow through a {@link Cleaner}. Everything else is removed.Start with one of the defaults:
@@ -53,15 +53,20 @@ If you need to allow more through (please be careful!), tweak a base safelist wi
- The cleaner and these safelists assume that you want to clean a body fragment of HTML (to add user
+ The {@link Cleaner} and these safelists assume that you want to clean a body fragment of HTML (to add user
supplied HTML into a templated page), and not to clean a full HTML document. If the latter is the case, you could wrap
the templated document HTML around the cleaned body HTML.
+ Safelists are mutable. A {@link Cleaner} uses the supplied safelist directly, so later changes affect later cleaning + calls. If you want to share a safelist across threads, finish configuring it first and do not mutate it while it is in + use. To build a variant from an existing configuration, use {@link #Safelist(Safelist)} to make a copy. +
+If you are going to extend a safelist, please be very careful. Make sure you understand what attributes may lead to XSS attack vectors. URL attributes are particularly vulnerable and require careful validation. See the XSS Filter Evasion Cheat Sheet for some - XSS attack examples (that jsoup will safegaurd against the default Cleaner and Safelist configuration). + XSS attack examples (that jsoup will safeguard against with the default Cleaner and Safelist configuration).
*/ public class Safelist { diff --git a/src/main/java/org/jsoup/select/NodeTraversor.java b/src/main/java/org/jsoup/select/NodeTraversor.java index b29afd0941..32ac44194f 100644 --- a/src/main/java/org/jsoup/select/NodeTraversor.java +++ b/src/main/java/org/jsoup/select/NodeTraversor.java @@ -10,7 +10,7 @@ order. The {@link NodeVisitor#head(Node, int)} and {@link NodeVisitor#tail(Node, int)} methods will be called for each node.During the head() visit, DOM structural changes around the node currently being visited are
- supported, including e.g. {@link Node#replaceWith(Node)} and {@link Node#remove()}. See
+ supported, including {@link Node#replaceWith(Node)} and {@link Node#remove()}. See
{@link NodeVisitor#head(Node, int) head()} for the traversal contract after mutation. Other non-structural node
changes are also supported.
DOM structural changes to the current node are not supported during the tail() visit.
The node may be modified (for example via {@link Node#attr(String)}), removed
Traversal never advances outside the original root subtree. If the traversal root is detached during + {@code head()}, traversal stops at the original root boundary.
@param node the node being visited. @param depth the depth of the node, relative to the root node. E.g., the root node has depth 0, and a child node diff --git a/src/test/java/org/jsoup/helper/DataUtilTest.java b/src/test/java/org/jsoup/helper/DataUtilTest.java index f557d2895c..1936871bc3 100644 --- a/src/test/java/org/jsoup/helper/DataUtilTest.java +++ b/src/test/java/org/jsoup/helper/DataUtilTest.java @@ -418,4 +418,25 @@ public int available() { } } + @Test + void charsetSniffingIgnoresAdvisoryAvailableIOException() throws IOException { + // https://github.com/jhy/jsoup/issues/2474 + // JDK 8's HttpURLConnection stream may throw from available() once the peer has closed the socket; + // that advisory failure does not mean we can't still consume bytes already buffered or read to clean EOF. + String html = "child
a
b
"); + Element root = doc.expectFirst("div"); + StringBuilder headOrder = new StringBuilder(); + StringBuilder tailOrder = new StringBuilder(); + + NodeTraversor.traverse(new NodeVisitor() { + @Override public void head(Node node, int depth) { + trackSeen(node, headOrder); + if (node == root) + node.remove(); + } + + @Override public void tail(Node node, int depth) { + trackSeen(node, tailOrder); + } + }, root); + + assertEquals("div;", headOrder.toString()); + assertEquals("", tailOrder.toString()); + assertEquals("b
", TextUtil.stripNewlines(doc.body().html())); + } + + @Test + void removingOnlyTraversalRootInHeadStopsWhenOriginalSlotIsEmpty() { + Document doc = Jsoup.parseBodyFragment("a
");
+ Element root = doc.expectFirst("div");
+ StringBuilder headOrder = new StringBuilder();
+ StringBuilder tailOrder = new StringBuilder();
+
+ NodeTraversor.traverse(new NodeVisitor() {
+ @Override public void head(Node node, int depth) {
+ trackSeen(node, headOrder);
+ if (node == root)
+ node.remove();
+ }
+
+ @Override public void tail(Node node, int depth) {
+ trackSeen(node, tailOrder);
+ }
+ }, root);
+
+ assertEquals("div;", headOrder.toString());
+ assertEquals("", tailOrder.toString());
+ assertEquals("", TextUtil.stripNewlines(doc.body().html()));
+ }
+
+ @Test
+ void replacingTraversalRootInHeadStaysWithinReplacementSubtree() {
+ Document doc = Jsoup.parseBodyFragment("x
y
"); + Element root = doc.expectFirst("div"); + StringBuilder headOrder = new StringBuilder(); + StringBuilder tailOrder = new StringBuilder(); + + NodeTraversor.traverse(new NodeVisitor() { + @Override public void head(Node node, int depth) { + trackSeen(node, headOrder); + if (node == root) { + Element replacement = new Element("section").insertChildren(0, node.childNodes()); + node.replaceWith(replacement); + } + } + + @Override public void tail(Node node, int depth) { + trackSeen(node, tailOrder); + } + }, root); + + assertEquals("div;span;x;", headOrder.toString()); + assertEquals("x;span;section;", tailOrder.toString()); + assertEquals("y
", TextUtil.stripNewlines(doc.body().html())); + } + + @Test + void unwrappingTraversalRootInHeadVisitsExposedTopLevelNodesUntilOriginalBoundary() { + Document doc = Jsoup.parseBodyFragment("xy
z
"); + Element root = doc.expectFirst("div"); + StringBuilder headOrder = new StringBuilder(); + StringBuilder tailOrder = new StringBuilder(); + + NodeTraversor.traverse(new NodeVisitor() { + @Override public void head(Node node, int depth) { + trackSeen(node, headOrder); + if (node == root) + node.unwrap(); + } + + @Override public void tail(Node node, int depth) { + trackSeen(node, tailOrder); + } + }, root); + + assertEquals("div;x;i;y;", headOrder.toString()); + assertEquals("x;b;y;i;", tailOrder.toString()); + assertEquals("xyz
", TextUtil.stripNewlines(doc.body().html())); + } + + @Test + void unwrapInHeadContinuesFromExposedChildren() { + Document doc = Jsoup.parseBodyFragment("xyz
");
+ Element root = doc.expectFirst("div");
+ StringBuilder headOrder = new StringBuilder();
+ StringBuilder tailOrder = new StringBuilder();
+
+ NodeTraversor.traverse(new NodeVisitor() {
+ @Override public void head(Node node, int depth) {
+ trackSeen(node, headOrder);
+ if (node instanceof Element && node.nameIs("span"))
+ node.unwrap();
+ }
+
+ @Override public void tail(Node node, int depth) {
+ trackSeen(node, tailOrder);
+ }
+ }, root);
+
+ assertEquals("div;span;x;i;y;u;z;", headOrder.toString());
+ assertEquals("x;b;y;i;z;u;div;", tailOrder.toString());
+ assertEquals("xyz
", TextUtil.stripNewlines(doc.body().html()));
+ }
+
+ @Test
+ void removingCurrentAndOriginalNextInHeadTailsParent() {
+ Document doc = Jsoup.parseBodyFragment("aTwo
");
+ Element root = doc.expectFirst("div");
+ StringBuilder headOrder = new StringBuilder();
+ StringBuilder tailOrder = new StringBuilder();
+
+ NodeTraversor.traverse(new NodeVisitor() {
+ @Override public void head(Node node, int depth) {
+ trackSeen(node, headOrder);
+ if (node instanceof TextNode && ((TextNode) node).text().equals("a")) {
+ node.nextSibling().remove();
+ node.remove();
+ }
+ }
+
+ @Override public void tail(Node node, int depth) {
+ trackSeen(node, tailOrder);
+ }
+ }, root);
+
+ assertEquals("div;a;", headOrder.toString());
+ assertEquals("div;", tailOrder.toString());
+ assertEquals("", TextUtil.stripNewlines(doc.body().html()));
+ }
+
+ @Test
+ void beforeAfterRemoveInHeadTailsCurrentSlotAndHeadsFutureSiblings() {
+ Document doc = Jsoup.parse("One
aTwoOne
bcTwo1
2
3"); Element body = doc.body();