1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 package org.htmlunit.html.parser.neko;
16
17 import java.io.IOException;
18 import java.io.InputStream;
19 import java.io.StringReader;
20 import java.net.URL;
21 import java.nio.charset.Charset;
22 import java.util.ArrayList;
23 import java.util.List;
24 import java.util.Map;
25 import java.util.concurrent.ConcurrentHashMap;
26
27 import org.htmlunit.ObjectInstantiationException;
28 import org.htmlunit.Page;
29 import org.htmlunit.SgmlPage;
30 import org.htmlunit.WebAssert;
31 import org.htmlunit.WebClient;
32 import org.htmlunit.WebResponse;
33 import org.htmlunit.cyberneko.HTMLScanner;
34 import org.htmlunit.cyberneko.HTMLTagBalancer;
35 import org.htmlunit.cyberneko.xerces.util.DefaultErrorHandler;
36 import org.htmlunit.cyberneko.xerces.xni.QName;
37 import org.htmlunit.cyberneko.xerces.xni.XNIException;
38 import org.htmlunit.cyberneko.xerces.xni.parser.XMLErrorHandler;
39 import org.htmlunit.cyberneko.xerces.xni.parser.XMLInputSource;
40 import org.htmlunit.cyberneko.xerces.xni.parser.XMLParseException;
41 import org.htmlunit.html.DefaultElementFactory;
42 import org.htmlunit.html.DomNode;
43 import org.htmlunit.html.ElementFactory;
44 import org.htmlunit.html.Html;
45 import org.htmlunit.html.HtmlPage;
46 import org.htmlunit.html.UnknownElementFactory;
47 import org.htmlunit.html.XHtmlPage;
48 import org.htmlunit.html.parser.HTMLParser;
49 import org.htmlunit.html.parser.HTMLParserListener;
50 import org.htmlunit.svg.SvgElementFactory;
51 import org.htmlunit.util.StringUtils;
52 import org.w3c.dom.Node;
53 import org.xml.sax.SAXException;
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70 public final class HtmlUnitNekoHtmlParser implements HTMLParser {
71
72
73
74
75 public static final SvgElementFactory SVG_FACTORY = new SvgElementFactory();
76
77 private static final Map<String, ElementFactory> ELEMENT_FACTORIES = new ConcurrentHashMap<>();
78
79 static {
80 final DefaultElementFactory defaultElementFactory = new DefaultElementFactory();
81 for (final String tagName : DefaultElementFactory.SUPPORTED_TAGS_) {
82 ELEMENT_FACTORIES.put(tagName, defaultElementFactory);
83 }
84 }
85
86
87
88
89 @Override
90 public void parseFragment(final WebClient webClient, final DomNode parent, final DomNode context,
91 final String source, final boolean createdByJavascript)
92 throws SAXException, IOException {
93 final Page page = parent.getPage();
94 if (!(page instanceof HtmlPage htmlPage)) {
95 return;
96 }
97 final URL url = htmlPage.getUrl();
98
99 final HtmlUnitNekoDOMBuilder domBuilder =
100 new HtmlUnitNekoDOMBuilder(this, webClient, parent, url, source, createdByJavascript);
101 domBuilder.setFeature("http://cyberneko.org/html/features/balance-tags/document-fragment", true);
102
103 final boolean xhtml = page instanceof XHtmlPage;
104 domBuilder.setFeature(HTMLScanner.ALLOW_SELFCLOSING_TAGS, xhtml);
105 domBuilder.setFeature(HTMLScanner.ALLOW_SELFCLOSING_IFRAME, xhtml);
106 domBuilder.setFeature(HTMLScanner.ALLOW_SELFCLOSING_SCRIPT, xhtml);
107
108
109 DomNode node = context;
110 final List<QName> ancestors = new ArrayList<>();
111 while (node != null && node.getNodeType() != Node.DOCUMENT_NODE) {
112 ancestors.add(0, new QName(null, node.getNodeName(), null, null));
113 node = node.getParentNode();
114 }
115 if (ancestors.isEmpty() || !"html".equals(ancestors.get(0).getLocalpart())) {
116 ancestors.add(new QName(null, "html", null, null));
117 ancestors.add(new QName(null, "body", null, null));
118 }
119 else if (ancestors.size() == 1
120 || (!"body".equals(ancestors.get(1).getLocalpart())
121 && !"head".equals(ancestors.get(1).getLocalpart()))) {
122 ancestors.add(new QName(null, "body", null, null));
123 }
124
125 domBuilder.setProperty(HTMLTagBalancer.FRAGMENT_CONTEXT_STACK, ancestors.toArray(new QName[0]));
126
127 final XMLInputSource in = new XMLInputSource(null, url.toString(), null, new StringReader(source), null);
128
129 htmlPage.registerParsingStart();
130 htmlPage.registerSnippetParsingStart();
131 try {
132 domBuilder.parse(in);
133 }
134 finally {
135 htmlPage.registerParsingEnd();
136 htmlPage.registerSnippetParsingEnd();
137 }
138 }
139
140
141
142
143 @Override
144 public void parse(final WebClient webClient, final WebResponse webResponse, final HtmlPage page,
145 final boolean xhtml, final boolean createdByJavascript) throws IOException {
146 final URL url = webResponse.getWebRequest().getUrl();
147 final HtmlUnitNekoDOMBuilder domBuilder =
148 new HtmlUnitNekoDOMBuilder(this, webClient, page, url, null, createdByJavascript);
149
150 final Charset charset = webResponse.getContentCharset();
151 try {
152 if (!webResponse.wasContentCharsetTentative()) {
153
154 domBuilder.setFeature(HTMLScanner.IGNORE_SPECIFIED_CHARSET, true);
155 }
156
157
158 if (xhtml) {
159 domBuilder.setFeature(HTMLScanner.ALLOW_SELFCLOSING_TAGS, true);
160 domBuilder.setFeature(HTMLScanner.ALLOW_SELFCLOSING_IFRAME, true);
161 domBuilder.setFeature(HTMLScanner.ALLOW_SELFCLOSING_SCRIPT, true);
162
163 domBuilder.setFeature(HTMLScanner.SCRIPT_STRIP_CDATA_DELIMS, true);
164 domBuilder.setFeature(HTMLScanner.STYLE_STRIP_CDATA_DELIMS, true);
165 domBuilder.setFeature(HTMLScanner.CDATA_SECTIONS, true);
166 domBuilder.setFeature(HTMLScanner.CDATA_EARLY_CLOSING, false);
167 }
168
169 final int bufferSize = webClient.getOptions().getNekoReaderBufferSize();
170 if (bufferSize > 0) {
171 domBuilder.setProperty(HTMLScanner.READER_BUFFER_SIZE, bufferSize);
172 }
173 }
174 catch (final Exception e) {
175 throw new ObjectInstantiationException("Error setting HTML parser feature", e);
176 }
177
178 try (InputStream content = webResponse.getContentAsStream()) {
179 final String encoding = charset.name();
180 final XMLInputSource in = new XMLInputSource(null, url.toString(), null, content, encoding);
181
182 page.registerParsingStart();
183 try {
184 domBuilder.parse(in);
185 }
186 catch (final XNIException e) {
187
188 final Throwable origin = extractNestedException(e);
189 throw new RuntimeException("Failed parsing content from " + url, origin);
190 }
191 }
192 finally {
193 page.registerParsingEnd();
194 }
195 }
196
197
198
199
200
201
202
203 static Throwable extractNestedException(final Throwable e) {
204 Throwable originalException;
205 Throwable cause = e;
206 do {
207 originalException = cause;
208
209 if (cause instanceof XNIException) {
210 cause = cause.getCause();
211 }
212 else {
213 cause = null;
214 }
215 }
216 while (cause != null);
217
218 return originalException;
219 }
220
221
222
223
224 @Override
225 public ElementFactory getSvgFactory() {
226 return SVG_FACTORY;
227 }
228
229
230
231
232 @Override
233 public ElementFactory getFactory(final String tagName) {
234 final ElementFactory result = ELEMENT_FACTORIES.get(tagName);
235
236 if (result != null) {
237 return result;
238 }
239 return UnknownElementFactory.INSTANCE;
240 }
241
242
243
244
245
246
247
248
249
250
251
252
253 @Override
254 public ElementFactory getElementFactory(final SgmlPage page, final String namespaceURI,
255 final String qualifiedName, final boolean insideSvg, final boolean svgSupport) {
256 if (insideSvg) {
257 return SVG_FACTORY;
258 }
259
260 if (namespaceURI == null || namespaceURI.isEmpty()
261 || Html.XHTML_NAMESPACE.equals(namespaceURI)
262 || Html.SVG_NAMESPACE.equals(namespaceURI)
263 || !qualifiedName.contains(":")) {
264
265 String tagName = qualifiedName;
266 final int index = tagName.indexOf(':');
267 if (index == -1) {
268 tagName = StringUtils.toRootLowerCase(tagName);
269 }
270 else {
271 tagName = tagName.substring(index + 1);
272 }
273 final ElementFactory factory;
274 if (svgSupport && !"svg".equals(tagName) && Html.SVG_NAMESPACE.equals(namespaceURI)) {
275 factory = SVG_FACTORY;
276 }
277 else {
278 factory = ELEMENT_FACTORIES.get(tagName);
279 }
280
281 if (factory != null) {
282 return factory;
283 }
284 }
285 return UnknownElementFactory.INSTANCE;
286 }
287 }
288
289
290
291
292 class HtmlUnitNekoHTMLErrorHandler implements XMLErrorHandler {
293 private final HTMLParserListener listener_;
294 private final URL url_;
295 private final String html_;
296
297 HtmlUnitNekoHTMLErrorHandler(final HTMLParserListener listener, final URL url, final String htmlContent) {
298 WebAssert.notNull("listener", listener);
299 WebAssert.notNull("url", url);
300 listener_ = listener;
301 url_ = url;
302 html_ = htmlContent;
303 }
304
305
306
307
308
309 @Override
310 public void error(final String domain, final String key,
311 final XMLParseException exception) throws XNIException {
312 listener_.error(exception.getMessage(),
313 url_,
314 html_,
315 exception.getLineNumber(),
316 exception.getColumnNumber(),
317 key);
318 }
319
320
321
322
323
324 @Override
325 public void warning(final String domain, final String key,
326 final XMLParseException exception) throws XNIException {
327 listener_.warning(exception.getMessage(),
328 url_,
329 html_,
330 exception.getLineNumber(),
331 exception.getColumnNumber(),
332 key);
333 }
334
335 @Override
336 public void fatalError(final String domain, final String key,
337 final XMLParseException exception) throws XNIException {
338 listener_.error(exception.getMessage(),
339 url_,
340 html_,
341 exception.getLineNumber(),
342 exception.getColumnNumber(),
343 key);
344 }
345 }