View Javadoc
1   /*
2    * Copyright (c) 2002-2025 Gargoyle Software Inc.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    * https://www.apache.org/licenses/LICENSE-2.0
8    *
9    * Unless required by applicable law or agreed to in writing, software
10   * distributed under the License is distributed on an "AS IS" BASIS,
11   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12   * See the License for the specific language governing permissions and
13   * limitations under the License.
14   */
15  package org.htmlunit.html;
16  
17  import static org.htmlunit.BrowserVersionFeatures.EVENT_FOCUS_ON_LOAD;
18  import static org.htmlunit.html.DomElement.ATTRIBUTE_NOT_DEFINED;
19  
20  import java.io.File;
21  import java.io.IOException;
22  import java.io.ObjectInputStream;
23  import java.io.ObjectOutputStream;
24  import java.io.Serializable;
25  import java.net.MalformedURLException;
26  import java.net.URL;
27  import java.nio.charset.Charset;
28  import java.nio.charset.StandardCharsets;
29  import java.util.ArrayList;
30  import java.util.Arrays;
31  import java.util.Collection;
32  import java.util.Collections;
33  import java.util.Comparator;
34  import java.util.HashMap;
35  import java.util.HashSet;
36  import java.util.Iterator;
37  import java.util.LinkedHashSet;
38  import java.util.List;
39  import java.util.Locale;
40  import java.util.Map;
41  import java.util.Set;
42  import java.util.WeakHashMap;
43  import java.util.concurrent.ConcurrentHashMap;
44  
45  import org.apache.commons.lang3.StringUtils;
46  import org.apache.commons.logging.Log;
47  import org.apache.commons.logging.LogFactory;
48  import org.htmlunit.Cache;
49  import org.htmlunit.ElementNotFoundException;
50  import org.htmlunit.FailingHttpStatusCodeException;
51  import org.htmlunit.History;
52  import org.htmlunit.HttpHeader;
53  import org.htmlunit.OnbeforeunloadHandler;
54  import org.htmlunit.Page;
55  import org.htmlunit.ScriptResult;
56  import org.htmlunit.SgmlPage;
57  import org.htmlunit.TopLevelWindow;
58  import org.htmlunit.WebAssert;
59  import org.htmlunit.WebClient;
60  import org.htmlunit.WebClientOptions;
61  import org.htmlunit.WebRequest;
62  import org.htmlunit.WebResponse;
63  import org.htmlunit.WebWindow;
64  import org.htmlunit.corejs.javascript.Function;
65  import org.htmlunit.corejs.javascript.Script;
66  import org.htmlunit.corejs.javascript.Scriptable;
67  import org.htmlunit.css.ComputedCssStyleDeclaration;
68  import org.htmlunit.css.CssStyleSheet;
69  import org.htmlunit.html.impl.SimpleRange;
70  import org.htmlunit.html.parser.HTMLParserDOMBuilder;
71  import org.htmlunit.http.HttpStatus;
72  import org.htmlunit.javascript.AbstractJavaScriptEngine;
73  import org.htmlunit.javascript.HtmlUnitScriptable;
74  import org.htmlunit.javascript.JavaScriptEngine;
75  import org.htmlunit.javascript.PostponedAction;
76  import org.htmlunit.javascript.host.Window;
77  import org.htmlunit.javascript.host.event.BeforeUnloadEvent;
78  import org.htmlunit.javascript.host.event.Event;
79  import org.htmlunit.javascript.host.event.EventTarget;
80  import org.htmlunit.javascript.host.html.HTMLDocument;
81  import org.htmlunit.protocol.javascript.JavaScriptURLConnection;
82  import org.htmlunit.util.MimeType;
83  import org.htmlunit.util.SerializableLock;
84  import org.htmlunit.util.UrlUtils;
85  import org.w3c.dom.Attr;
86  import org.w3c.dom.Comment;
87  import org.w3c.dom.DOMConfiguration;
88  import org.w3c.dom.DOMException;
89  import org.w3c.dom.DOMImplementation;
90  import org.w3c.dom.Document;
91  import org.w3c.dom.DocumentType;
92  import org.w3c.dom.Element;
93  import org.w3c.dom.EntityReference;
94  import org.w3c.dom.ProcessingInstruction;
95  
96  /**
97   * A representation of an HTML page returned from a server.
98   * <p>
99   * This class provides different methods to access the page's content like
100  * {@link #getForms()}, {@link #getAnchors()}, {@link #getElementById(String)}, ... as well as the
101  * very powerful inherited methods {@link #getByXPath(String)} and {@link #getFirstByXPath(String)}
102  * for fine grained user specific access to child nodes.
103  * </p>
104  * <p>
105  * Child elements allowing user interaction provide methods for this purpose like {@link HtmlAnchor#click()},
106  * {@link HtmlInput#type(String)}, {@link HtmlOption#setSelected(boolean)}, ...
107  * </p>
108  * <p>
109  * HtmlPage instances should not be instantiated directly. They will be returned by {@link WebClient#getPage(String)}
110  * when the content type of the server's response is <code>text/html</code> (or one of its variations).<br>
111  * <br>
112  * <b>Example:</b><br>
113  * <br>
114  * <code>
115  * final HtmlPage page = webClient.{@link WebClient#getPage(String) getPage}("http://mywebsite/some/page.html");
116  * </code>
117  * </p>
118  *
119  * @author <a href="mailto:mbowler@GargoyleSoftware.com">Mike Bowler</a>
120  * @author Alex Nikiforoff
121  * @author Noboru Sinohara
122  * @author David K. Taylor
123  * @author Andreas Hangler
124  * @author <a href="mailto:cse@dynabean.de">Christian Sell</a>
125  * @author Chris Erskine
126  * @author Marc Guillemot
127  * @author Ahmed Ashour
128  * @author Daniel Gredler
129  * @author Dmitri Zoubkov
130  * @author Sudhan Moghe
131  * @author Ethan Glasser-Camp
132  * @author <a href="mailto:tom.anderson@univ.oxon.org">Tom Anderson</a>
133  * @author Ronald Brill
134  * @author Frank Danek
135  * @author Joerg Werner
136  * @author Atsushi Nakagawa
137  * @author Rural Hunter
138  * @author Ronny Shapiro
139  * @author Lai Quang Duong
140  * @author Sven Strickroth
141  */
142 @SuppressWarnings("PMD.TooManyFields")
143 public class HtmlPage extends SgmlPage {
144 
145     private static final Log LOG = LogFactory.getLog(HtmlPage.class);
146 
147     private static final Comparator<DomElement> DOCUMENT_POSITION_COMPERATOR = new DocumentPositionComparator();
148 
149     private HTMLParserDOMBuilder domBuilder_;
150     private transient Charset originalCharset_;
151     private final Object lock_ = new SerializableLock(); // used for synchronization
152 
153     private Map<String, MappedElementIndexEntry> idMap_ = new ConcurrentHashMap<>();
154     private Map<String, MappedElementIndexEntry> nameMap_ = new ConcurrentHashMap<>();
155 
156     private List<BaseFrameElement> frameElements_ = new ArrayList<>();
157     private int parserCount_;
158     private int snippetParserCount_;
159     private int inlineSnippetParserCount_;
160     private Collection<HtmlAttributeChangeListener> attributeListeners_;
161     private List<PostponedAction> afterLoadActions_ = Collections.synchronizedList(new ArrayList<>());
162     private boolean cleaning_;
163     private HtmlBase base_;
164     private URL baseUrl_;
165     private List<AutoCloseable> autoCloseableList_;
166     private ElementFromPointHandler elementFromPointHandler_;
167     private DomElement elementWithFocus_;
168     private List<SimpleRange> selectionRanges_ = new ArrayList<>(3);
169 
170     private transient ComputedStylesCache computedStylesCache_;
171 
172     private static final HashSet<String> TABBABLE_TAGS =
173             new HashSet<>(Arrays.asList(HtmlAnchor.TAG_NAME, HtmlArea.TAG_NAME,
174                     HtmlButton.TAG_NAME, HtmlInput.TAG_NAME, HtmlObject.TAG_NAME,
175                     HtmlSelect.TAG_NAME, HtmlTextArea.TAG_NAME));
176     private static final HashSet<String> ACCEPTABLE_TAG_NAMES =
177             new HashSet<>(Arrays.asList(HtmlAnchor.TAG_NAME, HtmlArea.TAG_NAME,
178                     HtmlButton.TAG_NAME, HtmlInput.TAG_NAME, HtmlLabel.TAG_NAME,
179                     HtmlLegend.TAG_NAME, HtmlTextArea.TAG_NAME));
180 
181     /** Definition of special cases for the smart DomHtmlAttributeChangeListenerImpl */
182     private static final Set<String> ATTRIBUTES_AFFECTING_PARENT = new HashSet<>(Arrays.asList(
183             "style",
184             "class",
185             "height",
186             "width"));
187 
188     static class DocumentPositionComparator implements Comparator<DomElement>, Serializable {
189         @Override
190         public int compare(final DomElement elt1, final DomElement elt2) {
191             final short relation = elt1.compareDocumentPosition(elt2);
192             if (relation == 0) {
193                 return 0; // same node
194             }
195             if ((relation & DOCUMENT_POSITION_CONTAINS) != 0 || (relation & DOCUMENT_POSITION_PRECEDING) != 0) {
196                 return 1;
197             }
198 
199             return -1;
200         }
201     }
202 
203     /**
204      * Creates an instance of HtmlPage.
205      * An HtmlPage instance is normally retrieved with {@link WebClient#getPage(String)}.
206      *
207      * @param webResponse the web response that was used to create this page
208      * @param webWindow the window that this page is being loaded into
209      */
210     public HtmlPage(final WebResponse webResponse, final WebWindow webWindow) {
211         super(webResponse, webWindow);
212     }
213 
214     /**
215      * {@inheritDoc}
216      */
217     @Override
218     public HtmlPage getPage() {
219         return this;
220     }
221 
222     /**
223      * {@inheritDoc}
224      */
225     @Override
226     public boolean hasCaseSensitiveTagNames() {
227         return false;
228     }
229 
230     /**
231      * Initialize this page.
232      * @throws IOException if an IO problem occurs
233      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
234      *         {@link org.htmlunit.WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set
235      *         to true.
236      */
237     @Override
238     public void initialize() throws IOException, FailingHttpStatusCodeException {
239         final WebWindow enclosingWindow = getEnclosingWindow();
240         final boolean isAboutBlank = getUrl() == UrlUtils.URL_ABOUT_BLANK;
241         if (isAboutBlank) {
242             // a frame contains first a faked "about:blank" before its real content specified by src gets loaded
243             if (enclosingWindow instanceof FrameWindow
244                     && !((FrameWindow) enclosingWindow).getFrameElement().isContentLoaded()) {
245                 return;
246             }
247 
248             // save the URL that should be used to resolve relative URLs in this page
249             if (enclosingWindow instanceof TopLevelWindow) {
250                 final TopLevelWindow topWindow = (TopLevelWindow) enclosingWindow;
251                 final WebWindow openerWindow = topWindow.getOpener();
252                 if (openerWindow != null && openerWindow.getEnclosedPage() != null) {
253                     baseUrl_ = openerWindow.getEnclosedPage().getWebResponse().getWebRequest().getUrl();
254                 }
255             }
256         }
257 
258         if (!isAboutBlank) {
259             setReadyState(READY_STATE_INTERACTIVE);
260             getDocumentElement().setReadyState(READY_STATE_INTERACTIVE);
261             executeEventHandlersIfNeeded(Event.TYPE_READY_STATE_CHANGE);
262         }
263 
264         executeDeferredScriptsIfNeeded();
265 
266         executeEventHandlersIfNeeded(Event.TYPE_DOM_DOCUMENT_LOADED);
267 
268         loadFrames();
269 
270         // don't set the ready state if we really load the blank page into the window
271         // see Node.initInlineFrameIfNeeded()
272         if (!isAboutBlank) {
273             setReadyState(READY_STATE_COMPLETE);
274             getDocumentElement().setReadyState(READY_STATE_COMPLETE);
275             executeEventHandlersIfNeeded(Event.TYPE_READY_STATE_CHANGE);
276         }
277 
278         // frame initialization has a different order
279         boolean isFrameWindow = enclosingWindow instanceof FrameWindow;
280         boolean isFirstPageInFrameWindow = false;
281         if (isFrameWindow) {
282             isFrameWindow = ((FrameWindow) enclosingWindow).getFrameElement() instanceof HtmlFrame;
283 
284             final History hist = enclosingWindow.getHistory();
285             if (hist.getLength() > 0 && UrlUtils.URL_ABOUT_BLANK == hist.getUrl(0)) {
286                 isFirstPageInFrameWindow = hist.getLength() <= 2;
287             }
288             else {
289                 isFirstPageInFrameWindow = enclosingWindow.getHistory().getLength() < 2;
290             }
291         }
292 
293         if (isFrameWindow && !isFirstPageInFrameWindow) {
294             executeEventHandlersIfNeeded(Event.TYPE_LOAD);
295         }
296 
297         for (final BaseFrameElement frameElement : new ArrayList<>(frameElements_)) {
298             if (frameElement instanceof HtmlFrame) {
299                 final Page page = frameElement.getEnclosedWindow().getEnclosedPage();
300                 if (page != null && page.isHtmlPage()) {
301                     ((HtmlPage) page).executeEventHandlersIfNeeded(Event.TYPE_LOAD);
302                 }
303             }
304         }
305 
306         if (!isFrameWindow) {
307             executeEventHandlersIfNeeded(Event.TYPE_LOAD);
308 
309             if (!isAboutBlank && enclosingWindow.getWebClient().isJavaScriptEnabled()
310                     && hasFeature(EVENT_FOCUS_ON_LOAD)) {
311                 final HtmlElement body = getBody();
312                 if (body != null) {
313                     final Event event = new Event((Window) enclosingWindow.getScriptableObject(), Event.TYPE_FOCUS);
314                     body.fireEvent(event);
315                 }
316             }
317         }
318 
319         try {
320             while (!afterLoadActions_.isEmpty()) {
321                 final PostponedAction action = afterLoadActions_.remove(0);
322                 action.execute();
323             }
324         }
325         catch (final IOException e) {
326             throw e;
327         }
328         catch (final Exception e) {
329             throw new RuntimeException(e);
330         }
331         executeRefreshIfNeeded();
332     }
333 
334     /**
335      * Adds an action that should be executed once the page has been loaded.
336      * @param action the action
337      */
338     void addAfterLoadAction(final PostponedAction action) {
339         afterLoadActions_.add(action);
340     }
341 
342     /**
343      * Clean up this page.
344      */
345     @Override
346     public void cleanUp() {
347         //To avoid endless recursion caused by window.close() in onUnload
348         if (cleaning_) {
349             return;
350         }
351 
352         cleaning_ = true;
353         try {
354             super.cleanUp();
355             executeEventHandlersIfNeeded(Event.TYPE_UNLOAD);
356             deregisterFramesIfNeeded();
357         }
358         finally {
359             cleaning_ = false;
360 
361             if (autoCloseableList_ != null) {
362                 for (final AutoCloseable closeable : new ArrayList<>(autoCloseableList_)) {
363                     try {
364                         closeable.close();
365                     }
366                     catch (final Exception e) {
367                         LOG.error("Closing the autoclosable " + closeable + " failed", e);
368                     }
369                 }
370             }
371         }
372     }
373 
374     /**
375      * {@inheritDoc}
376      */
377     @Override
378     public HtmlElement getDocumentElement() {
379         return (HtmlElement) super.getDocumentElement();
380     }
381 
382     /**
383      * @return the <code>body</code> element, or {@code null} if it does not yet exist
384      */
385     public HtmlBody getBody() {
386         final DomElement doc = getDocumentElement();
387         if (doc != null) {
388             for (final DomNode node : doc.getChildren()) {
389                 if (node instanceof HtmlBody) {
390                     return (HtmlBody) node;
391                 }
392             }
393         }
394         return null;
395     }
396 
397     /**
398      * Returns the head element.
399      * @return the head element
400      */
401     public HtmlElement getHead() {
402         final DomElement doc = getDocumentElement();
403         if (doc != null) {
404             for (final DomNode node : doc.getChildren()) {
405                 if (node instanceof HtmlHead) {
406                     return (HtmlElement) node;
407                 }
408             }
409         }
410         return null;
411     }
412 
413     /**
414      * {@inheritDoc}
415      */
416     @Override
417     public Document getOwnerDocument() {
418         return null;
419     }
420 
421     /**
422      * {@inheritDoc}
423      * Not yet implemented.
424      */
425     @Override
426     public org.w3c.dom.Node importNode(final org.w3c.dom.Node importedNode, final boolean deep) {
427         throw new UnsupportedOperationException("HtmlPage.importNode is not yet implemented.");
428     }
429 
430     /**
431      * {@inheritDoc}
432      * Not yet implemented.
433      */
434     @Override
435     public String getInputEncoding() {
436         throw new UnsupportedOperationException("HtmlPage.getInputEncoding is not yet implemented.");
437     }
438 
439     /**
440      * {@inheritDoc}
441      */
442     @Override
443     public String getXmlEncoding() {
444         return null;
445     }
446 
447     /**
448      * {@inheritDoc}
449      */
450     @Override
451     public boolean getXmlStandalone() {
452         return false;
453     }
454 
455     /**
456      * {@inheritDoc}
457      * Not yet implemented.
458      */
459     @Override
460     public void setXmlStandalone(final boolean xmlStandalone) throws DOMException {
461         throw new UnsupportedOperationException("HtmlPage.setXmlStandalone is not yet implemented.");
462     }
463 
464     /**
465      * {@inheritDoc}
466      */
467     @Override
468     public String getXmlVersion() {
469         return null;
470     }
471 
472     /**
473      * {@inheritDoc}
474      * Not yet implemented.
475      */
476     @Override
477     public void setXmlVersion(final String xmlVersion) throws DOMException {
478         throw new UnsupportedOperationException("HtmlPage.setXmlVersion is not yet implemented.");
479     }
480 
481     /**
482      * {@inheritDoc}
483      * Not yet implemented.
484      */
485     @Override
486     public boolean getStrictErrorChecking() {
487         throw new UnsupportedOperationException("HtmlPage.getStrictErrorChecking is not yet implemented.");
488     }
489 
490     /**
491      * {@inheritDoc}
492      * Not yet implemented.
493      */
494     @Override
495     public void setStrictErrorChecking(final boolean strictErrorChecking) {
496         throw new UnsupportedOperationException("HtmlPage.setStrictErrorChecking is not yet implemented.");
497     }
498 
499     /**
500      * {@inheritDoc}
501      * Not yet implemented.
502      */
503     @Override
504     public String getDocumentURI() {
505         throw new UnsupportedOperationException("HtmlPage.getDocumentURI is not yet implemented.");
506     }
507 
508     /**
509      * {@inheritDoc}
510      * Not yet implemented.
511      */
512     @Override
513     public void setDocumentURI(final String documentURI) {
514         throw new UnsupportedOperationException("HtmlPage.setDocumentURI is not yet implemented.");
515     }
516 
517     /**
518      * {@inheritDoc}
519      * Not yet implemented.
520      */
521     @Override
522     public org.w3c.dom.Node adoptNode(final org.w3c.dom.Node source) throws DOMException {
523         throw new UnsupportedOperationException("HtmlPage.adoptNode is not yet implemented.");
524     }
525 
526     /**
527      * {@inheritDoc}
528      * Not yet implemented.
529      */
530     @Override
531     public DOMConfiguration getDomConfig() {
532         throw new UnsupportedOperationException("HtmlPage.getDomConfig is not yet implemented.");
533     }
534 
535     /**
536      * {@inheritDoc}
537      * Not yet implemented.
538      */
539     @Override
540     public org.w3c.dom.Node renameNode(final org.w3c.dom.Node newNode, final String namespaceURI,
541         final String qualifiedName) throws DOMException {
542         throw new UnsupportedOperationException("HtmlPage.renameNode is not yet implemented.");
543     }
544 
545     /**
546      * {@inheritDoc}
547      */
548     @Override
549     public Charset getCharset() {
550         if (originalCharset_ == null) {
551             originalCharset_ = getWebResponse().getContentCharset();
552         }
553         return originalCharset_;
554     }
555 
556     /**
557      * {@inheritDoc}
558      */
559     @Override
560     public String getContentType() {
561         return getWebResponse().getContentType();
562     }
563 
564     /**
565      * {@inheritDoc}
566      * Not yet implemented.
567      */
568     @Override
569     public DOMImplementation getImplementation() {
570         throw new UnsupportedOperationException("HtmlPage.getImplementation is not yet implemented.");
571     }
572 
573     /**
574      * {@inheritDoc}
575      * @param tagName the tag name, preferably in lowercase
576      */
577     @Override
578     public DomElement createElement(String tagName) {
579         if (tagName.indexOf(':') == -1) {
580             tagName = org.htmlunit.util.StringUtils.toRootLowerCase(tagName);
581         }
582         return getWebClient().getPageCreator().getHtmlParser().getFactory(tagName)
583                     .createElementNS(this, null, tagName, null);
584     }
585 
586     /**
587      * {@inheritDoc}
588      */
589     @Override
590     public DomElement createElementNS(final String namespaceURI, final String qualifiedName) {
591         return getWebClient().getPageCreator().getHtmlParser()
592                 .getElementFactory(this, namespaceURI, qualifiedName, false, true)
593                 .createElementNS(this, namespaceURI, qualifiedName, null);
594     }
595 
596     /**
597      * {@inheritDoc}
598      * Not yet implemented.
599      */
600     @Override
601     public Attr createAttributeNS(final String namespaceURI, final String qualifiedName) {
602         throw new UnsupportedOperationException("HtmlPage.createAttributeNS is not yet implemented.");
603     }
604 
605     /**
606      * {@inheritDoc}
607      * Not yet implemented.
608      */
609     @Override
610     public EntityReference createEntityReference(final String id) {
611         throw new UnsupportedOperationException("HtmlPage.createEntityReference is not yet implemented.");
612     }
613 
614     /**
615      * {@inheritDoc}
616      * Not yet implemented.
617      */
618     @Override
619     public ProcessingInstruction createProcessingInstruction(final String namespaceURI, final String qualifiedName) {
620         throw new UnsupportedOperationException("HtmlPage.createProcessingInstruction is not yet implemented.");
621     }
622 
623     /**
624      * {@inheritDoc}
625      */
626     @Override
627     public DomElement getElementById(final String elementId) {
628         if (elementId != null) {
629             final MappedElementIndexEntry elements = idMap_.get(elementId);
630             if (elements != null) {
631                 return elements.first();
632             }
633         }
634         return null;
635     }
636 
637     /**
638      * Returns the {@link HtmlAnchor} with the specified name.
639      *
640      * @param name the name to search by
641      * @return the {@link HtmlAnchor} with the specified name
642      * @throws ElementNotFoundException if the anchor could not be found
643      */
644     public HtmlAnchor getAnchorByName(final String name) throws ElementNotFoundException {
645         return getDocumentElement().getOneHtmlElementByAttribute("a", DomElement.NAME_ATTRIBUTE, name);
646     }
647 
648     /**
649      * Returns the {@link HtmlAnchor} with the specified href.
650      *
651      * @param href the string to search by
652      * @return the HtmlAnchor
653      * @throws ElementNotFoundException if the anchor could not be found
654      */
655     public HtmlAnchor getAnchorByHref(final String href) throws ElementNotFoundException {
656         return getDocumentElement().getOneHtmlElementByAttribute("a", "href", href);
657     }
658 
659     /**
660      * Returns a list of all anchors contained in this page.
661      * @return the list of {@link HtmlAnchor} in this page
662      */
663     public List<HtmlAnchor> getAnchors() {
664         return getDocumentElement().getElementsByTagNameImpl("a");
665     }
666 
667     /**
668      * Returns the first anchor with the specified text.
669      * @param text the text to search for
670      * @return the first anchor that was found
671      * @throws ElementNotFoundException if no anchors are found with the specified text
672      */
673     public HtmlAnchor getAnchorByText(final String text) throws ElementNotFoundException {
674         WebAssert.notNull("text", text);
675 
676         for (final HtmlAnchor anchor : getAnchors()) {
677             if (text.equals(anchor.asNormalizedText())) {
678                 return anchor;
679             }
680         }
681         throw new ElementNotFoundException("a", "<text>", text);
682     }
683 
684     /**
685      * Returns the first form that matches the specified name.
686      * @param name the name to search for
687      * @return the first form
688      * @exception ElementNotFoundException If no forms match the specified result.
689      */
690     public HtmlForm getFormByName(final String name) throws ElementNotFoundException {
691         final List<HtmlForm> forms = getDocumentElement()
692                 .getElementsByAttribute("form", DomElement.NAME_ATTRIBUTE, name);
693         if (forms.isEmpty()) {
694             throw new ElementNotFoundException("form", DomElement.NAME_ATTRIBUTE, name);
695         }
696         return forms.get(0);
697     }
698 
699     /**
700      * Returns a list of all the forms in this page.
701      * @return all the forms in this page
702      */
703     public List<HtmlForm> getForms() {
704         return getDocumentElement().getElementsByTagNameImpl("form");
705     }
706 
707     /**
708      * Given a relative URL (ie <code>/foo</code>), returns a fully-qualified URL based on
709      * the URL that was used to load this page.
710      *
711      * @param relativeUrl the relative URL
712      * @return the fully-qualified URL for the specified relative URL
713      * @exception MalformedURLException if an error occurred when creating a URL object
714      */
715     public URL getFullyQualifiedUrl(String relativeUrl) throws MalformedURLException {
716         // to handle http: and http:/ in FF (Bug #474)
717         boolean incorrectnessNotified = false;
718         while (relativeUrl.startsWith("http:") && !relativeUrl.startsWith("http://")) {
719             if (!incorrectnessNotified) {
720                 notifyIncorrectness("Incorrect URL \"" + relativeUrl + "\" has been corrected");
721                 incorrectnessNotified = true;
722             }
723             relativeUrl = "http:/" + relativeUrl.substring(5);
724         }
725 
726         return WebClient.expandUrl(getBaseURL(), relativeUrl);
727     }
728 
729     /**
730      * Given a target attribute value, resolve the target using a base target for the page.
731      *
732      * @param elementTarget the target specified as an attribute of the element
733      * @return the resolved target to use for the element
734      */
735     public String getResolvedTarget(final String elementTarget) {
736         final String resolvedTarget;
737         if (base_ == null) {
738             resolvedTarget = elementTarget;
739         }
740         else if (elementTarget != null && !elementTarget.isEmpty()) {
741             resolvedTarget = elementTarget;
742         }
743         else {
744             resolvedTarget = base_.getTargetAttribute();
745         }
746         return resolvedTarget;
747     }
748 
749     /**
750      * Returns a list of ids (strings) that correspond to the tabbable elements
751      * in this page. Return them in the same order specified in {@link #getTabbableElements}
752      *
753      * @return the list of id's
754      */
755     public List<String> getTabbableElementIds() {
756         final List<String> list = new ArrayList<>();
757 
758         for (final HtmlElement element : getTabbableElements()) {
759             list.add(element.getId());
760         }
761 
762         return Collections.unmodifiableList(list);
763     }
764 
765     /**
766      * Returns a list of all elements that are tabbable in the order that will
767      * be used for tabbing.<p>
768      *
769      * The rules for determining tab order are as follows:
770      * <ol>
771      *   <li>Those elements that support the tabindex attribute and assign a
772      *   positive value to it are navigated first. Navigation proceeds from the
773      *   element with the lowest tabindex value to the element with the highest
774      *   value. Values need not be sequential nor must they begin with any
775      *   particular value. Elements that have identical tabindex values should
776      *   be navigated in the order they appear in the character stream.
777      *   <li>Those elements that do not support the tabindex attribute or
778      *   support it and assign it a value of "0" are navigated next. These
779      *   elements are navigated in the order they appear in the character
780      *   stream.
781      *   <li>Elements that are disabled do not participate in the tabbing
782      *   order.
783      * </ol>
784      * Additionally, the value of tabindex must be within 0 and 32767. Any
785      * values outside this range will be ignored.<p>
786      *
787      * The following elements support the <code>tabindex</code> attribute:
788      * A, AREA, BUTTON, INPUT, OBJECT, SELECT, and TEXTAREA.
789      *
790      * @return all the tabbable elements in proper tab order
791      */
792     public List<HtmlElement> getTabbableElements() {
793         final List<HtmlElement> tabbableElements = new ArrayList<>();
794         for (final HtmlElement element : getHtmlElementDescendants()) {
795             final String tagName = element.getTagName();
796             if (TABBABLE_TAGS.contains(tagName)) {
797                 final boolean disabled = element.isDisabledElementAndDisabled();
798                 if (!disabled && !HtmlElement.TAB_INDEX_OUT_OF_BOUNDS.equals(element.getTabIndex())) {
799                     tabbableElements.add(element);
800                 }
801             }
802         }
803         tabbableElements.sort(createTabOrderComparator());
804         return Collections.unmodifiableList(tabbableElements);
805     }
806 
807     private static Comparator<HtmlElement> createTabOrderComparator() {
808         return (element1, element2) -> {
809             final Short i1 = element1.getTabIndex();
810             final Short i2 = element2.getTabIndex();
811 
812             final short index1;
813             if (i1 == null) {
814                 index1 = -1;
815             }
816             else {
817                 index1 = i1.shortValue();
818             }
819 
820             final short index2;
821             if (i2 == null) {
822                 index2 = -1;
823             }
824             else {
825                 index2 = i2.shortValue();
826             }
827 
828             final int result;
829             if (index1 > 0 && index2 > 0) {
830                 result = index1 - index2;
831             }
832             else if (index1 > 0) {
833                 result = -1;
834             }
835             else if (index2 > 0) {
836                 result = 1;
837             }
838             else if (index1 == index2) {
839                 result = 0;
840             }
841             else {
842                 result = index2 - index1;
843             }
844 
845             return result;
846         };
847     }
848 
849     /**
850      * Returns the HTML element that is assigned to the specified access key. An
851      * access key (aka mnemonic key) is used for keyboard navigation of the
852      * page.<p>
853      *
854      * Only the following HTML elements may have <code>accesskey</code>s defined: A, AREA,
855      * BUTTON, INPUT, LABEL, LEGEND, and TEXTAREA.
856      *
857      * @param accessKey the key to look for
858      * @return the HTML element that is assigned to the specified key or null
859      *      if no elements can be found that match the specified key.
860      */
861     public HtmlElement getHtmlElementByAccessKey(final char accessKey) {
862         final List<HtmlElement> elements = getHtmlElementsByAccessKey(accessKey);
863         if (elements.isEmpty()) {
864             return null;
865         }
866         return elements.get(0);
867     }
868 
869     /**
870      * Returns all the HTML elements that are assigned to the specified access key. An
871      * access key (aka mnemonic key) is used for keyboard navigation of the
872      * page.<p>
873      *
874      * The HTML specification seems to indicate that one accesskey cannot be used
875      * for multiple elements however Internet Explorer does seem to support this.
876      * It's worth noting that Firefox does not support multiple elements with one
877      * access key so you are making your HTML browser specific if you rely on this
878      * feature.<p>
879      *
880      * Only the following HTML elements may have <code>accesskey</code>s defined: A, AREA,
881      * BUTTON, INPUT, LABEL, LEGEND, and TEXTAREA.
882      *
883      * @param accessKey the key to look for
884      * @return the elements that are assigned to the specified accesskey
885      */
886     public List<HtmlElement> getHtmlElementsByAccessKey(final char accessKey) {
887         final List<HtmlElement> elements = new ArrayList<>();
888 
889         final String searchString = Character.toString(accessKey).toLowerCase(Locale.ROOT);
890         for (final HtmlElement element : getHtmlElementDescendants()) {
891             if (ACCEPTABLE_TAG_NAMES.contains(element.getTagName())) {
892                 final String accessKeyAttribute = element.getAttributeDirect("accesskey");
893                 if (searchString.equalsIgnoreCase(accessKeyAttribute)) {
894                     elements.add(element);
895                 }
896             }
897         }
898 
899         return elements;
900     }
901 
902     /**
903      * <p>Executes the specified JavaScript code within the page. The usage would be similar to what can
904      * be achieved to execute JavaScript in the current page by entering "javascript:...some JS code..."
905      * in the URL field of a native browser.</p>
906      * <p><b>Note:</b> the provided code won't be executed if JavaScript has been disabled on the WebClient
907      * (see {@link org.htmlunit.WebClient#isJavaScriptEnabled()}.</p>
908      * @param sourceCode the JavaScript code to execute
909      * @return a ScriptResult which will contain both the current page (which may be different than
910      *         the previous page) and a JavaScript result object
911      */
912     public ScriptResult executeJavaScript(final String sourceCode) {
913         return executeJavaScript(sourceCode, "injected script", 1);
914     }
915 
916     /**
917      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
918      * <p>
919      * Execute the specified JavaScript if a JavaScript engine was successfully
920      * instantiated. If this JavaScript causes the current page to be reloaded
921      * (through location="" or form.submit()) then return the new page. Otherwise
922      * return the current page.
923      * </p>
924      * <p><b>Please note:</b> Although this method is public, it is not intended for
925      * general execution of JavaScript. Users of HtmlUnit should interact with the pages
926      * as a user would by clicking on buttons or links and having the JavaScript event
927      * handlers execute as needed.
928      * </p>
929      *
930      * @param sourceCode the JavaScript code to execute
931      * @param sourceName the name for this chunk of code (will be displayed in error messages)
932      * @param startLine the line at which the script source starts
933      * @return a ScriptResult which will contain both the current page (which may be different than
934      *         the previous page and a JavaScript result object.
935      */
936     public ScriptResult executeJavaScript(String sourceCode, final String sourceName, final int startLine) {
937         if (!getWebClient().isJavaScriptEnabled()) {
938             return new ScriptResult(JavaScriptEngine.UNDEFINED);
939         }
940 
941         if (StringUtils.startsWithIgnoreCase(sourceCode, JavaScriptURLConnection.JAVASCRIPT_PREFIX)) {
942             sourceCode = sourceCode.substring(JavaScriptURLConnection.JAVASCRIPT_PREFIX.length()).trim();
943             if (sourceCode.startsWith("return ")) {
944                 sourceCode = sourceCode.substring("return ".length());
945             }
946         }
947 
948         final Object result = getWebClient().getJavaScriptEngine()
949                 .execute(this, getEnclosingWindow().getScriptableObject(), sourceCode, sourceName, startLine);
950         return new ScriptResult(result);
951     }
952 
953     /** Various possible external JavaScript file loading results. */
954     enum JavaScriptLoadResult {
955         /** The load was aborted and nothing was done. */
956         NOOP,
957         /** The load was aborted and nothing was done. */
958         NO_CONTENT,
959         /** The external JavaScript file was downloaded and compiled successfully. */
960         SUCCESS,
961         /** The external JavaScript file was not downloaded successfully. */
962         DOWNLOAD_ERROR,
963         /** The external JavaScript file was downloaded but was not compiled successfully. */
964         COMPILATION_ERROR
965     }
966 
967     /**
968      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
969      *
970      * @param srcAttribute the source attribute from the script tag
971      * @param scriptCharset the charset from the script tag
972      * @return the result of loading the specified external JavaScript file
973      * @throws FailingHttpStatusCodeException if the request's status code indicates a request
974      *         failure and the {@link WebClient} was configured to throw exceptions on failing
975      *         HTTP status codes
976      */
977     JavaScriptLoadResult loadExternalJavaScriptFile(final String srcAttribute, final Charset scriptCharset)
978         throws FailingHttpStatusCodeException {
979 
980         final WebClient client = getWebClient();
981         if (StringUtils.isBlank(srcAttribute) || !client.isJavaScriptEnabled()) {
982             return JavaScriptLoadResult.NOOP;
983         }
984 
985         final URL scriptURL;
986         try {
987             scriptURL = getFullyQualifiedUrl(srcAttribute);
988             final String protocol = scriptURL.getProtocol();
989             if ("javascript".equals(protocol)) {
990                 if (LOG.isInfoEnabled()) {
991                     LOG.info("Ignoring script src [" + srcAttribute + "]");
992                 }
993                 return JavaScriptLoadResult.NOOP;
994             }
995             if (!"http".equals(protocol) && !"https".equals(protocol)
996                     && !"data".equals(protocol) && !"file".equals(protocol)) {
997                 client.getJavaScriptErrorListener().malformedScriptURL(this, srcAttribute,
998                         new MalformedURLException("unknown protocol: '" + protocol + "'"));
999                 return JavaScriptLoadResult.NOOP;
1000             }
1001         }
1002         catch (final MalformedURLException e) {
1003             client.getJavaScriptErrorListener().malformedScriptURL(this, srcAttribute, e);
1004             return JavaScriptLoadResult.NOOP;
1005         }
1006 
1007         final Object script;
1008         try {
1009             script = loadJavaScriptFromUrl(scriptURL, scriptCharset);
1010         }
1011         catch (final IOException e) {
1012             client.getJavaScriptErrorListener().loadScriptError(this, scriptURL, e);
1013             return JavaScriptLoadResult.DOWNLOAD_ERROR;
1014         }
1015         catch (final FailingHttpStatusCodeException e) {
1016             if (e.getStatusCode() == HttpStatus.NO_CONTENT_204) {
1017                 return JavaScriptLoadResult.NO_CONTENT;
1018             }
1019             client.getJavaScriptErrorListener().loadScriptError(this, scriptURL, e);
1020             throw e;
1021         }
1022 
1023         if (script == null) {
1024             return JavaScriptLoadResult.COMPILATION_ERROR;
1025         }
1026 
1027         @SuppressWarnings("unchecked")
1028         final AbstractJavaScriptEngine<Object> engine = (AbstractJavaScriptEngine<Object>) client.getJavaScriptEngine();
1029         engine.execute(this, getEnclosingWindow().getScriptableObject(), script);
1030         return JavaScriptLoadResult.SUCCESS;
1031     }
1032 
1033     /**
1034      * Loads JavaScript from the specified URL. This method may return {@code null} if
1035      * there is a problem loading the code from the specified URL.
1036      *
1037      * @param url the URL of the script
1038      * @param scriptCharset the charset from the script tag
1039      * @return the content of the file, or {@code null} if we ran into a compile error
1040      * @throws IOException if there is a problem downloading the JavaScript file
1041      * @throws FailingHttpStatusCodeException if the request's status code indicates a request
1042      *         failure and the {@link WebClient} was configured to throw exceptions on failing
1043      *         HTTP status codes
1044      */
1045     private Object loadJavaScriptFromUrl(final URL url, final Charset scriptCharset) throws IOException,
1046         FailingHttpStatusCodeException {
1047 
1048         final WebRequest referringRequest = getWebResponse().getWebRequest();
1049 
1050         final WebClient client = getWebClient();
1051         final WebRequest request = new WebRequest(url);
1052         // copy all headers from the referring request
1053         request.setAdditionalHeaders(new HashMap<>(referringRequest.getAdditionalHeaders()));
1054 
1055         // at least overwrite this headers
1056         request.setAdditionalHeader(HttpHeader.ACCEPT, client.getBrowserVersion().getScriptAcceptHeader());
1057         request.setAdditionalHeader(HttpHeader.SEC_FETCH_SITE, "same-origin");
1058         request.setAdditionalHeader(HttpHeader.SEC_FETCH_MODE, "no-cors");
1059         request.setAdditionalHeader(HttpHeader.SEC_FETCH_DEST, "script");
1060 
1061         request.setRefererHeader(referringRequest.getUrl());
1062         request.setCharset(scriptCharset);
1063 
1064         // use info from script tag or fall back to utf-8
1065         // https://www.rfc-editor.org/rfc/rfc9239#section-4.2
1066         if (scriptCharset != null) {
1067             request.setDefaultResponseContentCharset(scriptCharset);
1068         }
1069         else {
1070             request.setDefaultResponseContentCharset(StandardCharsets.UTF_8);
1071         }
1072 
1073         // our cache is a bit strange;
1074         // loadWebResponse check the cache for the web response
1075         // AND also fixes the request url for the following cache lookups
1076         final WebResponse response = client.loadWebResponse(request);
1077 
1078         // now we can look into the cache with the fixed request for
1079         // a cached script
1080         final Cache cache = client.getCache();
1081         final Object cachedScript = cache.getCachedObject(request);
1082         if (cachedScript instanceof Script) {
1083             return cachedScript;
1084         }
1085 
1086         client.printContentIfNecessary(response);
1087         client.throwFailingHttpStatusCodeExceptionIfNecessary(response);
1088 
1089         final int statusCode = response.getStatusCode();
1090         if (statusCode == HttpStatus.NO_CONTENT_204) {
1091             throw new FailingHttpStatusCodeException(response);
1092         }
1093 
1094         if (!response.isSuccess()) {
1095             throw new IOException("Unable to download JavaScript from '" + url + "' (status " + statusCode + ").");
1096         }
1097 
1098         final String contentType = response.getContentType();
1099         if (contentType != null) {
1100             if (MimeType.isObsoleteJavascriptMimeType(contentType)) {
1101                 getWebClient().getIncorrectnessListener().notify(
1102                         "Obsolete content type encountered: '" + contentType + "' "
1103                                 + "for remotely loaded JavaScript element at '" + url + "'.", this);
1104             }
1105             else if (!MimeType.isJavascriptMimeType(contentType)) {
1106                 getWebClient().getIncorrectnessListener().notify(
1107                         "Expect content type of '" + MimeType.TEXT_JAVASCRIPT + "' "
1108                                 + "for remotely loaded JavaScript element at '" + url + "', "
1109                                 + "but got '" + contentType + "'.", this);
1110             }
1111         }
1112 
1113         final Charset scriptEncoding = response.getContentCharset();
1114         final String scriptCode = response.getContentAsString(scriptEncoding);
1115         if (null != scriptCode) {
1116             final AbstractJavaScriptEngine<?> javaScriptEngine = client.getJavaScriptEngine();
1117             final Scriptable scope = getEnclosingWindow().getScriptableObject();
1118             final Object script = javaScriptEngine.compile(this, scope, scriptCode, url.toExternalForm(), 1);
1119             if (script != null && cache.cacheIfPossible(request, response, script)) {
1120                 // no cleanup if the response is stored inside the cache
1121                 return script;
1122             }
1123 
1124             response.cleanUp();
1125             return script;
1126         }
1127 
1128         response.cleanUp();
1129         return null;
1130     }
1131 
1132     /**
1133      * Returns the title of this page or an empty string if the title wasn't specified.
1134      *
1135      * @return the title of this page or an empty string if the title wasn't specified
1136      */
1137     public String getTitleText() {
1138         final HtmlTitle titleElement = getTitleElement();
1139         if (titleElement != null) {
1140             return titleElement.asNormalizedText();
1141         }
1142         return "";
1143     }
1144 
1145     /**
1146      * Sets the text for the title of this page. If there is not a title element
1147      * on this page, then one has to be generated.
1148      * @param message the new text
1149      */
1150     public void setTitleText(final String message) {
1151         HtmlTitle titleElement = getTitleElement();
1152         if (titleElement == null) {
1153             LOG.debug("No title element, creating one");
1154             final HtmlHead head = (HtmlHead) getFirstChildElement(getDocumentElement(), HtmlHead.class);
1155             if (head == null) {
1156                 // perhaps should we create head too?
1157                 throw new IllegalStateException("Headelement was not defined for this page");
1158             }
1159             final Map<String, DomAttr> emptyMap = Collections.emptyMap();
1160             titleElement = new HtmlTitle(HtmlTitle.TAG_NAME, this, emptyMap);
1161             if (head.getFirstChild() != null) {
1162                 head.getFirstChild().insertBefore(titleElement);
1163             }
1164             else {
1165                 head.appendChild(titleElement);
1166             }
1167         }
1168 
1169         titleElement.setNodeValue(message);
1170     }
1171 
1172     /**
1173      * Gets the first child of startElement that is an instance of the given class.
1174      * @param startElement the parent element
1175      * @param clazz the class to search for
1176      * @return {@code null} if no child found
1177      */
1178     private static DomElement getFirstChildElement(final DomElement startElement, final Class<?> clazz) {
1179         if (startElement == null) {
1180             return null;
1181         }
1182         for (final DomElement element : startElement.getChildElements()) {
1183             if (clazz.isInstance(element)) {
1184                 return element;
1185             }
1186         }
1187 
1188         return null;
1189     }
1190 
1191     /**
1192      * Gets the first child of startElement or it's children that is an instance of the given class.
1193      * @param startElement the parent element
1194      * @param clazz the class to search for
1195      * @return {@code null} if no child found
1196      */
1197     private DomElement getFirstChildElementRecursive(final DomElement startElement, final Class<?> clazz) {
1198         if (startElement == null) {
1199             return null;
1200         }
1201         for (final DomElement element : startElement.getChildElements()) {
1202             if (clazz.isInstance(element)) {
1203                 return element;
1204             }
1205             final DomElement childFound = getFirstChildElementRecursive(element, clazz);
1206             if (childFound != null) {
1207                 return childFound;
1208             }
1209         }
1210 
1211         return null;
1212     }
1213 
1214     /**
1215      * Gets the title element for this page. Returns null if one is not found.
1216      *
1217      * @return the title element for this page or null if this is not one
1218      */
1219     private HtmlTitle getTitleElement() {
1220         return (HtmlTitle) getFirstChildElementRecursive(getDocumentElement(), HtmlTitle.class);
1221     }
1222 
1223     /**
1224      * Looks for and executes any appropriate event handlers. Looks for body and frame tags.
1225      * @param eventType either {@link Event#TYPE_LOAD}, {@link Event#TYPE_UNLOAD}, or {@link Event#TYPE_BEFORE_UNLOAD}
1226      * @return {@code true} if user accepted <code>onbeforeunload</code> (not relevant to other events)
1227      */
1228     private boolean executeEventHandlersIfNeeded(final String eventType) {
1229         // If JavaScript isn't enabled, there's nothing for us to do.
1230         if (!getWebClient().isJavaScriptEnabled()) {
1231             return true;
1232         }
1233 
1234         // Execute the specified event on the document element.
1235         final WebWindow window = getEnclosingWindow();
1236         if (window.getScriptableObject() instanceof Window) {
1237             final Event event;
1238             if (Event.TYPE_BEFORE_UNLOAD.equals(eventType)) {
1239                 event = new BeforeUnloadEvent(this, eventType);
1240             }
1241             else {
1242                 event = new Event(this, eventType);
1243             }
1244 
1245             // This is the same as DomElement.fireEvent() and was copied
1246             // here so it could be used with HtmlPage.
1247             if (LOG.isDebugEnabled()) {
1248                 LOG.debug("Firing " + event);
1249             }
1250 
1251             final EventTarget jsNode;
1252             if (Event.TYPE_DOM_DOCUMENT_LOADED.equals(eventType)) {
1253                 jsNode = getScriptableObject();
1254             }
1255             else if (Event.TYPE_READY_STATE_CHANGE.equals(eventType)) {
1256                 jsNode = getDocumentElement().getScriptableObject();
1257             }
1258             else {
1259                 // The load/beforeunload/unload events target Document but paths Window only (tested in Chrome/FF)
1260                 jsNode = window.getScriptableObject();
1261             }
1262 
1263             ((JavaScriptEngine) getWebClient().getJavaScriptEngine()).callSecured(cx -> jsNode.fireEvent(event), this);
1264 
1265             if (!isOnbeforeunloadAccepted(this, event)) {
1266                 return false;
1267             }
1268         }
1269 
1270         // If this page was loaded in a frame, execute the version of the event specified on the frame tag.
1271         if (window instanceof FrameWindow) {
1272             final FrameWindow fw = (FrameWindow) window;
1273             final BaseFrameElement frame = fw.getFrameElement();
1274 
1275             // if part of a document fragment, then the load event is not triggered
1276             if (Event.TYPE_LOAD.equals(eventType) && frame.getParentNode() instanceof DomDocumentFragment) {
1277                 return true;
1278             }
1279 
1280             if (frame.hasEventHandlers("on" + eventType)) {
1281                 if (LOG.isDebugEnabled()) {
1282                     LOG.debug("Executing on" + eventType + " handler for " + frame);
1283                 }
1284                 if (window.getScriptableObject() instanceof Window) {
1285                     final Event event;
1286                     if (Event.TYPE_BEFORE_UNLOAD.equals(eventType)) {
1287                         event = new BeforeUnloadEvent(frame, eventType);
1288                     }
1289                     else {
1290                         event = new Event(frame, eventType);
1291                     }
1292                     // This fires the "load" event for the <frame> element which, like all non-window
1293                     // load events, propagates up to Document but not Window.  The "load" event for
1294                     // <frameset> on the other hand, like that of <body>, is handled above where it is
1295                     // fired against Document and directed to Window.
1296                     frame.fireEvent(event);
1297 
1298                     if (!isOnbeforeunloadAccepted((HtmlPage) frame.getPage(), event)) {
1299                         return false;
1300                     }
1301                 }
1302             }
1303         }
1304 
1305         return true;
1306     }
1307 
1308     /**
1309      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1310      *
1311      * @return true if the OnbeforeunloadHandler has accepted to change the page
1312      */
1313     public boolean isOnbeforeunloadAccepted() {
1314         return executeEventHandlersIfNeeded(Event.TYPE_BEFORE_UNLOAD);
1315     }
1316 
1317     private boolean isOnbeforeunloadAccepted(final HtmlPage page, final Event event) {
1318         if (event instanceof BeforeUnloadEvent) {
1319             final BeforeUnloadEvent beforeUnloadEvent = (BeforeUnloadEvent) event;
1320             if (beforeUnloadEvent.isBeforeUnloadMessageSet()) {
1321                 final OnbeforeunloadHandler handler = getWebClient().getOnbeforeunloadHandler();
1322                 if (handler == null) {
1323                     LOG.warn("document.onbeforeunload() returned a string in event.returnValue,"
1324                             + " but no onbeforeunload handler installed.");
1325                 }
1326                 else {
1327                     final String message = JavaScriptEngine.toString(beforeUnloadEvent.getReturnValue());
1328                     return handler.handleEvent(page, message);
1329                 }
1330             }
1331         }
1332         return true;
1333     }
1334 
1335     /**
1336      * If a refresh has been specified either through a meta tag or an HTTP
1337      * response header, then perform that refresh.
1338      * @throws IOException if an IO problem occurs
1339      */
1340     private void executeRefreshIfNeeded() throws IOException {
1341         // If this page is not in a frame then a refresh has already happened,
1342         // most likely through the JavaScript onload handler, so we don't do a
1343         // second refresh.
1344         final WebWindow window = getEnclosingWindow();
1345         if (window == null) {
1346             return;
1347         }
1348 
1349         final String refreshString = getRefreshStringOrNull();
1350         if (refreshString == null || refreshString.isEmpty()) {
1351             return;
1352         }
1353 
1354         final double time;
1355         final URL url;
1356 
1357         int index = StringUtils.indexOfAnyBut(refreshString, "0123456789");
1358         final boolean timeOnly = index == -1;
1359 
1360         if (timeOnly) {
1361             // Format: <meta http-equiv='refresh' content='10'>
1362             try {
1363                 time = Double.parseDouble(refreshString);
1364             }
1365             catch (final NumberFormatException e) {
1366                 if (LOG.isErrorEnabled()) {
1367                     LOG.error("Malformed refresh string (no ';' but not a number): " + refreshString, e);
1368                 }
1369                 return;
1370             }
1371             url = getUrl();
1372         }
1373         else {
1374             // Format: <meta http-equiv='refresh' content='10;url=http://www.blah.com'>
1375             try {
1376                 time = Double.parseDouble(refreshString.substring(0, index).trim());
1377             }
1378             catch (final NumberFormatException e) {
1379                 if (LOG.isErrorEnabled()) {
1380                     LOG.error("Malformed refresh string (no valid number before ';') " + refreshString, e);
1381                 }
1382                 return;
1383             }
1384             index = refreshString.toLowerCase(Locale.ROOT).indexOf("url=", index);
1385             if (index == -1) {
1386                 if (LOG.isErrorEnabled()) {
1387                     LOG.error("Malformed refresh string (found ';' but no 'url='): " + refreshString);
1388                 }
1389                 return;
1390             }
1391             final StringBuilder builder = new StringBuilder(refreshString.substring(index + 4));
1392             if (StringUtils.isBlank(builder.toString())) {
1393                 //content='10; URL=' is treated as content='10'
1394                 url = getUrl();
1395             }
1396             else {
1397                 if (builder.charAt(0) == '"' || builder.charAt(0) == 0x27) {
1398                     builder.deleteCharAt(0);
1399                 }
1400                 if (builder.charAt(builder.length() - 1) == '"' || builder.charAt(builder.length() - 1) == 0x27) {
1401                     builder.deleteCharAt(builder.length() - 1);
1402                 }
1403                 final String urlString = builder.toString();
1404                 try {
1405                     url = getFullyQualifiedUrl(urlString);
1406                 }
1407                 catch (final MalformedURLException e) {
1408                     if (LOG.isErrorEnabled()) {
1409                         LOG.error("Malformed URL in refresh string: " + refreshString, e);
1410                     }
1411                     throw e;
1412                 }
1413             }
1414         }
1415 
1416         final int timeRounded = (int) time;
1417         checkRecursion();
1418         getWebClient().getRefreshHandler().handleRefresh(this, url, timeRounded);
1419     }
1420 
1421     private void checkRecursion() {
1422         final StackTraceElement[] elements = new Exception().getStackTrace();
1423         if (elements.length > 500) {
1424             for (int i = 0; i < 500; i++) {
1425                 if (!elements[i].getClassName().startsWith("org.htmlunit.")) {
1426                     return;
1427                 }
1428             }
1429             final WebResponse webResponse = getWebResponse();
1430             throw new FailingHttpStatusCodeException("Too much redirect for "
1431                     + webResponse.getWebRequest().getUrl(), webResponse);
1432         }
1433     }
1434 
1435     /**
1436      * Returns an auto-refresh string if specified. This will look in both the meta
1437      * tags and inside the HTTP response headers.
1438      * @return the auto-refresh string
1439      */
1440     private String getRefreshStringOrNull() {
1441         final List<HtmlMeta> metaTags = getMetaTags("refresh");
1442         if (!metaTags.isEmpty()) {
1443             return metaTags.get(0).getContentAttribute().trim();
1444         }
1445         return getWebResponse().getResponseHeaderValue("Refresh");
1446     }
1447 
1448     /**
1449      * Executes any deferred scripts, if necessary.
1450      */
1451     private void executeDeferredScriptsIfNeeded() {
1452         if (!getWebClient().isJavaScriptEnabled()) {
1453             return;
1454         }
1455         final DomElement doc = getDocumentElement();
1456         final List<HtmlScript> scripts = new ArrayList<>();
1457 
1458         // don't call getElementsByTagName() here because it creates a live collection
1459         for (final HtmlElement elem : doc.getHtmlElementDescendants()) {
1460             if ("script".equals(elem.getLocalName()) && (elem instanceof HtmlScript)) {
1461                 final HtmlScript script = (HtmlScript) elem;
1462                 if (script.isDeferred() && ATTRIBUTE_NOT_DEFINED != script.getSrcAttribute()) {
1463                     scripts.add(script);
1464                 }
1465             }
1466         }
1467         for (final HtmlScript script : scripts) {
1468             ScriptElementSupport.executeScriptIfNeeded(script, true, true);
1469         }
1470     }
1471 
1472     /**
1473      * Deregister frames that are no longer in use.
1474      */
1475     public void deregisterFramesIfNeeded() {
1476         for (final BaseFrameElement frameElement : frameElements_) {
1477             final WebWindow window = frameElement.getEnclosedWindow();
1478             getWebClient().deregisterWebWindow(window);
1479             final Page page = window.getEnclosedPage();
1480             if (page != null && page.isHtmlPage()) {
1481                 // seems quite silly, but for instance if the src attribute of an iframe is not
1482                 // set, the error only occurs when leaving the page
1483                 ((HtmlPage) page).deregisterFramesIfNeeded();
1484             }
1485         }
1486     }
1487 
1488     /**
1489      * Returns a list containing all the frames (from frame and iframe tags) in this page
1490      * in document order.
1491      * @return a list of {@link FrameWindow}
1492      */
1493     public List<FrameWindow> getFrames() {
1494         final List<BaseFrameElement> frameElements = new ArrayList<>(frameElements_);
1495         Collections.sort(frameElements, DOCUMENT_POSITION_COMPERATOR);
1496 
1497         final List<FrameWindow> list = new ArrayList<>(frameElements.size());
1498         for (final BaseFrameElement frameElement : frameElements) {
1499             list.add(frameElement.getEnclosedWindow());
1500         }
1501         return list;
1502     }
1503 
1504     /**
1505      * Returns the first frame contained in this page with the specified name.
1506      * @param name the name to search for
1507      * @return the first frame found
1508      * @exception ElementNotFoundException If no frame exist in this page with the specified name.
1509      */
1510     public FrameWindow getFrameByName(final String name) throws ElementNotFoundException {
1511         for (final BaseFrameElement frameElement : frameElements_) {
1512             final FrameWindow fw = frameElement.getEnclosedWindow();
1513             if (fw.getName().equals(name)) {
1514                 return fw;
1515             }
1516         }
1517 
1518         throw new ElementNotFoundException("frame or iframe", DomElement.NAME_ATTRIBUTE, name);
1519     }
1520 
1521     /**
1522      * Simulate pressing an access key. This may change the focus, may click buttons and may invoke
1523      * JavaScript.
1524      *
1525      * @param accessKey the key that will be pressed
1526      * @return the element that has the focus after pressing this access key or null if no element
1527      *         has the focus.
1528      * @throws IOException if an IO error occurs during the processing of this access key (this
1529      *         would only happen if the access key triggered a button which in turn caused a page load)
1530      */
1531     public DomElement pressAccessKey(final char accessKey) throws IOException {
1532         final HtmlElement element = getHtmlElementByAccessKey(accessKey);
1533         if (element != null) {
1534             element.focus();
1535             if (element instanceof HtmlAnchor
1536                     || element instanceof HtmlArea
1537                     || element instanceof HtmlButton
1538                     || element instanceof HtmlInput
1539                     || element instanceof HtmlLabel
1540                     || element instanceof HtmlLegend
1541                     || element instanceof HtmlTextArea) {
1542                 final Page newPage = element.click();
1543 
1544                 if (newPage != this && getFocusedElement() == element) {
1545                     // The page was reloaded therefore no element on this page will have the focus.
1546                     getFocusedElement().blur();
1547                 }
1548             }
1549         }
1550 
1551         return getFocusedElement();
1552     }
1553 
1554     /**
1555      * Move the focus to the next element in the tab order. To determine the specified tab
1556      * order, refer to {@link HtmlPage#getTabbableElements()}
1557      *
1558      * @return the element that has focus after calling this method
1559      */
1560     public HtmlElement tabToNextElement() {
1561         final List<HtmlElement> elements = getTabbableElements();
1562         if (elements.isEmpty()) {
1563             setFocusedElement(null);
1564             return null;
1565         }
1566 
1567         final HtmlElement elementToGiveFocus;
1568         final DomElement elementWithFocus = getFocusedElement();
1569         if (elementWithFocus == null) {
1570             elementToGiveFocus = elements.get(0);
1571         }
1572         else {
1573             final int index = elements.indexOf(elementWithFocus);
1574             if (index == -1) {
1575                 // The element with focus isn't on this page
1576                 elementToGiveFocus = elements.get(0);
1577             }
1578             else {
1579                 if (index == elements.size() - 1) {
1580                     elementToGiveFocus = elements.get(0);
1581                 }
1582                 else {
1583                     elementToGiveFocus = elements.get(index + 1);
1584                 }
1585             }
1586         }
1587 
1588         setFocusedElement(elementToGiveFocus);
1589         return elementToGiveFocus;
1590     }
1591 
1592     /**
1593      * Move the focus to the previous element in the tab order. To determine the specified tab
1594      * order, refer to {@link HtmlPage#getTabbableElements()}
1595      *
1596      * @return the element that has focus after calling this method
1597      */
1598     public HtmlElement tabToPreviousElement() {
1599         final List<HtmlElement> elements = getTabbableElements();
1600         if (elements.isEmpty()) {
1601             setFocusedElement(null);
1602             return null;
1603         }
1604 
1605         final HtmlElement elementToGiveFocus;
1606         final DomElement elementWithFocus = getFocusedElement();
1607         if (elementWithFocus == null) {
1608             elementToGiveFocus = elements.get(elements.size() - 1);
1609         }
1610         else {
1611             final int index = elements.indexOf(elementWithFocus);
1612             if (index == -1) {
1613                 // The element with focus isn't on this page
1614                 elementToGiveFocus = elements.get(elements.size() - 1);
1615             }
1616             else {
1617                 if (index == 0) {
1618                     elementToGiveFocus = elements.get(elements.size() - 1);
1619                 }
1620                 else {
1621                     elementToGiveFocus = elements.get(index - 1);
1622                 }
1623             }
1624         }
1625 
1626         setFocusedElement(elementToGiveFocus);
1627         return elementToGiveFocus;
1628     }
1629 
1630     /**
1631      * Returns the HTML element with the specified ID. If more than one element
1632      * has this ID (not allowed by the HTML spec), then this method returns the
1633      * first one.
1634      *
1635      * @param elementId the ID value to search for
1636      * @param <E> the element type
1637      * @return the HTML element with the specified ID
1638      * @throws ElementNotFoundException if no element was found matching the specified ID
1639      */
1640     @SuppressWarnings("unchecked")
1641     public <E extends HtmlElement> E getHtmlElementById(final String elementId) throws ElementNotFoundException {
1642         final DomElement element = getElementById(elementId);
1643         if (element == null) {
1644             throw new ElementNotFoundException("*", DomElement.ID_ATTRIBUTE, elementId);
1645         }
1646         return (E) element;
1647     }
1648 
1649     /**
1650      * Returns the elements with the specified ID. If there are no elements
1651      * with the specified ID, this method returns an empty list. Please note that
1652      * the lists returned by this method are immutable.
1653      *
1654      * @param elementId the ID value to search for
1655      * @return the elements with the specified name attribute
1656      */
1657     public List<DomElement> getElementsById(final String elementId) {
1658         if (elementId != null) {
1659             final MappedElementIndexEntry elements = idMap_.get(elementId);
1660             if (elements != null) {
1661                 return new ArrayList<>(elements.elements());
1662             }
1663         }
1664         return Collections.emptyList();
1665     }
1666 
1667     /**
1668      * Returns the element with the specified name. If more than one element
1669      * has this name, then this method returns the first one.
1670      *
1671      * @param name the name value to search for
1672      * @param <E> the element type
1673      * @return the element with the specified name
1674      * @throws ElementNotFoundException if no element was found matching the specified name
1675      */
1676     @SuppressWarnings("unchecked")
1677     public <E extends DomElement> E getElementByName(final String name) throws ElementNotFoundException {
1678         if (name != null) {
1679             final MappedElementIndexEntry elements = nameMap_.get(name);
1680             if (elements != null) {
1681                 return (E) elements.first();
1682             }
1683         }
1684         throw new ElementNotFoundException("*", DomElement.NAME_ATTRIBUTE, name);
1685     }
1686 
1687     /**
1688      * Returns the elements with the specified name attribute. If there are no elements
1689      * with the specified name, this method returns an empty list. Please note that
1690      * the lists returned by this method are immutable.
1691      *
1692      * @param name the name value to search for
1693      * @return the elements with the specified name attribute
1694      */
1695     public List<DomElement> getElementsByName(final String name) {
1696         if (name != null) {
1697             final MappedElementIndexEntry elements = nameMap_.get(name);
1698             if (elements != null) {
1699                 return new ArrayList<>(elements.elements());
1700             }
1701         }
1702         return Collections.emptyList();
1703     }
1704 
1705     /**
1706      * Returns the elements with the specified string for their name or ID. If there are
1707      * no elements with the specified name or ID, this method returns an empty list.
1708      *
1709      * @param idAndOrName the value to search for
1710      * @return the elements with the specified string for their name or ID
1711      */
1712     public List<DomElement> getElementsByIdAndOrName(final String idAndOrName) {
1713         if (idAndOrName == null) {
1714             return Collections.emptyList();
1715         }
1716         final MappedElementIndexEntry list1 = idMap_.get(idAndOrName);
1717         final MappedElementIndexEntry list2 = nameMap_.get(idAndOrName);
1718         final List<DomElement> list = new ArrayList<>();
1719         if (list1 != null) {
1720             list.addAll(list1.elements());
1721         }
1722         if (list2 != null) {
1723             for (final DomElement elt : list2.elements()) {
1724                 if (!list.contains(elt)) {
1725                     list.add(elt);
1726                 }
1727             }
1728         }
1729         return list;
1730     }
1731 
1732     /**
1733      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1734      *
1735      * @param node the node that has just been added to the document
1736      */
1737     void notifyNodeAdded(final DomNode node) {
1738         if (node instanceof DomElement) {
1739             addMappedElement((DomElement) node, true);
1740 
1741             if (node instanceof BaseFrameElement) {
1742                 frameElements_.add((BaseFrameElement) node);
1743             }
1744 
1745             if (node.getFirstChild() != null) {
1746                 for (final Iterator<HtmlElement> iterator = node.new DescendantHtmlElementsIterator();
1747                         iterator.hasNext();) {
1748                     final HtmlElement child = iterator.next();
1749                     if (child instanceof BaseFrameElement) {
1750                         frameElements_.add((BaseFrameElement) child);
1751                     }
1752                 }
1753             }
1754 
1755             if ("base".equals(node.getNodeName())) {
1756                 calculateBase();
1757             }
1758         }
1759         node.onAddedToPage();
1760     }
1761 
1762     /**
1763      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1764      *
1765      * @param node the node that has just been removed from the tree
1766      */
1767     void notifyNodeRemoved(final DomNode node) {
1768         if (node instanceof HtmlElement) {
1769             removeMappedElement((HtmlElement) node, true, true);
1770 
1771             if (node instanceof BaseFrameElement) {
1772                 frameElements_.remove(node);
1773             }
1774             for (final HtmlElement child : node.getHtmlElementDescendants()) {
1775                 if (child instanceof BaseFrameElement) {
1776                     frameElements_.remove(child);
1777                 }
1778             }
1779 
1780             if ("base".equals(node.getNodeName())) {
1781                 calculateBase();
1782             }
1783         }
1784     }
1785 
1786     /**
1787      * Adds an element to the ID and name maps, if necessary.
1788      * @param element the element to be added to the ID and name maps
1789      * @param recurse indicates if children must be added too
1790      */
1791     void addMappedElement(final DomElement element, final boolean recurse) {
1792         if (isAncestorOf(element)) {
1793             addElement(idMap_, element, DomElement.ID_ATTRIBUTE, recurse);
1794             addElement(nameMap_, element, DomElement.NAME_ATTRIBUTE, recurse);
1795         }
1796     }
1797 
1798     private void addElement(final Map<String, MappedElementIndexEntry> map, final DomElement element,
1799             final String attribute, final boolean recurse) {
1800         final String value = element.getAttribute(attribute);
1801 
1802         if (ATTRIBUTE_NOT_DEFINED != value) {
1803             MappedElementIndexEntry elements = map.get(value);
1804             if (elements == null) {
1805                 elements = new MappedElementIndexEntry();
1806                 elements.add(element);
1807                 map.put(value, elements);
1808             }
1809             else {
1810                 elements.add(element);
1811             }
1812         }
1813         if (recurse) {
1814             // poor man's approach - we don't use getChildElements()
1815             // to avoid a bunch of object constructions
1816             DomNode nextChild = element.getFirstChild();
1817             while (nextChild != null) {
1818                 if (nextChild instanceof DomElement) {
1819                     addElement(map, (DomElement) nextChild, attribute, true);
1820                 }
1821                 nextChild = nextChild.getNextSibling();
1822             }
1823         }
1824     }
1825 
1826     /**
1827      * Removes an element and optionally its children from the ID and name maps, if necessary.
1828      * @param element the element to be removed from the ID and name maps
1829      * @param recurse indicates if children must be removed too
1830      * @param descendant indicates of the element was descendant of this HtmlPage, but now its parent might be null
1831      */
1832     void removeMappedElement(final DomElement element, final boolean recurse, final boolean descendant) {
1833         if (descendant || isAncestorOf(element)) {
1834             removeElement(idMap_, element, DomElement.ID_ATTRIBUTE, recurse);
1835             removeElement(nameMap_, element, DomElement.NAME_ATTRIBUTE, recurse);
1836         }
1837     }
1838 
1839     private void removeElement(final Map<String, MappedElementIndexEntry> map, final DomElement element,
1840             final String attribute, final boolean recurse) {
1841         final String value = element.getAttribute(attribute);
1842 
1843         if (ATTRIBUTE_NOT_DEFINED != value) {
1844             final MappedElementIndexEntry elements = map.remove(value);
1845             if (elements != null) {
1846                 elements.remove(element);
1847                 if (!elements.elements_.isEmpty()) {
1848                     map.put(value, elements);
1849                 }
1850             }
1851         }
1852         if (recurse) {
1853             for (final DomElement child : element.getChildElements()) {
1854                 removeElement(map, child, attribute, true);
1855             }
1856         }
1857     }
1858 
1859     /**
1860      * Indicates if the attribute name indicates that the owning element is mapped.
1861      * @param document the owning document
1862      * @param attributeName the name of the attribute to consider
1863      * @return {@code true} if the owning element should be mapped in its owning page
1864      */
1865     static boolean isMappedElement(final Document document, final String attributeName) {
1866         return document instanceof HtmlPage
1867             && (DomElement.NAME_ATTRIBUTE.equals(attributeName) || DomElement.ID_ATTRIBUTE.equals(attributeName));
1868     }
1869 
1870     private void calculateBase() {
1871         final List<HtmlElement> baseElements = getDocumentElement().getStaticElementsByTagName("base");
1872 
1873         base_ = null;
1874         for (final HtmlElement baseElement : baseElements) {
1875             if (baseElement instanceof HtmlBase) {
1876                 if (base_ != null) {
1877                     notifyIncorrectness("Multiple 'base' detected, only the first is used.");
1878                     break;
1879                 }
1880                 base_ = (HtmlBase) baseElement;
1881             }
1882         }
1883     }
1884 
1885     /**
1886      * Loads the content of the contained frames. This is done after the page is completely loaded, to allow script
1887      * contained in the frames to reference elements from the page located after the closing &lt;/frame&gt; tag.
1888      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
1889      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to {@code true}
1890      */
1891     void loadFrames() throws FailingHttpStatusCodeException {
1892         for (final BaseFrameElement frameElement : new ArrayList<>(frameElements_)) {
1893             // test if the frame should really be loaded:
1894             // if a script has already changed its content, it should be skipped
1895             // use == and not equals(...) to identify initial content (versus URL set to "about:blank")
1896             if (frameElement.getEnclosedWindow() != null
1897                     && UrlUtils.URL_ABOUT_BLANK == frameElement.getEnclosedPage().getUrl()
1898                     && !frameElement.isContentLoaded()) {
1899                 frameElement.loadInnerPage();
1900             }
1901         }
1902     }
1903 
1904     /**
1905      * Gives a basic representation for debugging purposes.
1906      * @return a basic representation
1907      */
1908     @Override
1909     public String toString() {
1910         final StringBuilder builder = new StringBuilder()
1911             .append("HtmlPage(")
1912             .append(getUrl())
1913             .append(")@")
1914             .append(hashCode());
1915         return builder.toString();
1916     }
1917 
1918     /**
1919      * Gets the meta tag for a given {@code http-equiv} value.
1920      * @param httpEquiv the {@code http-equiv} value
1921      * @return a list of {@link HtmlMeta}
1922      */
1923     protected List<HtmlMeta> getMetaTags(final String httpEquiv) {
1924         if (getDocumentElement() == null) {
1925             return Collections.emptyList(); // weird case, for instance if document.documentElement has been removed
1926         }
1927         final List<HtmlMeta> tags = getDocumentElement().getStaticElementsByTagName("meta");
1928         final List<HtmlMeta> foundTags = new ArrayList<>();
1929         for (final HtmlMeta htmlMeta : tags) {
1930             if (httpEquiv.equalsIgnoreCase(htmlMeta.getHttpEquivAttribute())) {
1931                 foundTags.add(htmlMeta);
1932             }
1933         }
1934         return foundTags;
1935     }
1936 
1937     /**
1938      * Creates a clone of this instance, and clears cached state to be not shared with the original.
1939      *
1940      * @return a clone of this instance
1941      */
1942     @Override
1943     protected HtmlPage clone() {
1944         final HtmlPage result = (HtmlPage) super.clone();
1945         result.elementWithFocus_ = null;
1946 
1947         result.idMap_ = new ConcurrentHashMap<>();
1948         result.nameMap_ = new ConcurrentHashMap<>();
1949 
1950         return result;
1951     }
1952 
1953     /**
1954      * {@inheritDoc}
1955      */
1956     @Override
1957     public HtmlPage cloneNode(final boolean deep) {
1958         // we need the ScriptObject clone before cloning the kids.
1959         final HtmlPage result = (HtmlPage) super.cloneNode(false);
1960         if (getWebClient().isJavaScriptEnabled()) {
1961             final HtmlUnitScriptable jsObjClone = getScriptableObject().clone();
1962             jsObjClone.setDomNode(result);
1963         }
1964 
1965         // if deep, clone the kids too, and re initialize parts of the clone
1966         if (deep) {
1967             // this was previously synchronized but that makes not sense, why
1968             // lock the source against a copy only one has a reference too,
1969             // because result is a local reference
1970             result.attributeListeners_ = null;
1971 
1972             result.selectionRanges_ = new ArrayList<>(3);
1973             // the original one is synchronized so we should do that here too, shouldn't we?
1974             result.afterLoadActions_ = Collections.synchronizedList(new ArrayList<>());
1975             result.frameElements_ = new ArrayList<>();
1976             for (DomNode child = getFirstChild(); child != null; child = child.getNextSibling()) {
1977                 result.appendChild(child.cloneNode(true));
1978             }
1979         }
1980         return result;
1981     }
1982 
1983     /**
1984      * Adds an HtmlAttributeChangeListener to the listener list.
1985      * The listener is registered for all attributes of all HtmlElements contained in this page.
1986      *
1987      * @param listener the attribute change listener to be added
1988      * @see #removeHtmlAttributeChangeListener(HtmlAttributeChangeListener)
1989      */
1990     public void addHtmlAttributeChangeListener(final HtmlAttributeChangeListener listener) {
1991         WebAssert.notNull("listener", listener);
1992         synchronized (lock_) {
1993             if (attributeListeners_ == null) {
1994                 attributeListeners_ = new LinkedHashSet<>();
1995             }
1996             attributeListeners_.add(listener);
1997         }
1998     }
1999 
2000     /**
2001      * Removes an HtmlAttributeChangeListener from the listener list.
2002      * This method should be used to remove HtmlAttributeChangeListener that were registered
2003      * for all attributes of all HtmlElements contained in this page.
2004      *
2005      * @param listener the attribute change listener to be removed
2006      * @see #addHtmlAttributeChangeListener(HtmlAttributeChangeListener)
2007      */
2008     public void removeHtmlAttributeChangeListener(final HtmlAttributeChangeListener listener) {
2009         WebAssert.notNull("listener", listener);
2010         synchronized (lock_) {
2011             if (attributeListeners_ != null) {
2012                 attributeListeners_.remove(listener);
2013             }
2014         }
2015     }
2016 
2017     /**
2018      * Notifies all registered listeners for the given event to add an attribute.
2019      * @param event the event to fire
2020      */
2021     void fireHtmlAttributeAdded(final HtmlAttributeChangeEvent event) {
2022         final List<HtmlAttributeChangeListener> listeners = safeGetAttributeListeners();
2023         if (listeners != null) {
2024             for (final HtmlAttributeChangeListener listener : listeners) {
2025                 listener.attributeAdded(event);
2026             }
2027         }
2028     }
2029 
2030     /**
2031      * Notifies all registered listeners for the given event to replace an attribute.
2032      * @param event the event to fire
2033      */
2034     void fireHtmlAttributeReplaced(final HtmlAttributeChangeEvent event) {
2035         final List<HtmlAttributeChangeListener> listeners = safeGetAttributeListeners();
2036         if (listeners != null) {
2037             for (final HtmlAttributeChangeListener listener : listeners) {
2038                 listener.attributeReplaced(event);
2039             }
2040         }
2041     }
2042 
2043     /**
2044      * Notifies all registered listeners for the given event to remove an attribute.
2045      * @param event the event to fire
2046      */
2047     void fireHtmlAttributeRemoved(final HtmlAttributeChangeEvent event) {
2048         final List<HtmlAttributeChangeListener> listeners = safeGetAttributeListeners();
2049         if (listeners != null) {
2050             for (final HtmlAttributeChangeListener listener : listeners) {
2051                 listener.attributeRemoved(event);
2052             }
2053         }
2054     }
2055 
2056     private List<HtmlAttributeChangeListener> safeGetAttributeListeners() {
2057         synchronized (lock_) {
2058             if (attributeListeners_ != null) {
2059                 return new ArrayList<>(attributeListeners_);
2060             }
2061             return null;
2062         }
2063     }
2064 
2065     /**
2066      * {@inheritDoc}
2067      */
2068     @Override
2069     protected void checkChildHierarchy(final org.w3c.dom.Node newChild) throws DOMException {
2070         if (newChild instanceof Element) {
2071             if (getDocumentElement() != null) {
2072                 throw new DOMException(DOMException.HIERARCHY_REQUEST_ERR,
2073                     "The Document may only have a single child Element.");
2074             }
2075         }
2076         else if (newChild instanceof DocumentType) {
2077             if (getDoctype() != null) {
2078                 throw new DOMException(DOMException.HIERARCHY_REQUEST_ERR,
2079                     "The Document may only have a single child DocumentType.");
2080             }
2081         }
2082         else if (!(newChild instanceof Comment || newChild instanceof ProcessingInstruction)) {
2083             throw new DOMException(DOMException.HIERARCHY_REQUEST_ERR,
2084                 "The Document may not have a child of this type: " + newChild.getNodeType());
2085         }
2086         super.checkChildHierarchy(newChild);
2087     }
2088 
2089     /**
2090      * Returns {@code true} if an HTML parser is operating on this page, adding content to it.
2091      * @return {@code true} if an HTML parser is operating on this page, adding content to it
2092      */
2093     public boolean isBeingParsed() {
2094         return parserCount_ > 0;
2095     }
2096 
2097     /**
2098      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2099      *
2100      * Called by the HTML parser to let the page know that it has started parsing some content for this page.
2101      */
2102     public void registerParsingStart() {
2103         parserCount_++;
2104     }
2105 
2106     /**
2107      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2108      *
2109      * Called by the HTML parser to let the page know that it has finished parsing some content for this page.
2110      */
2111     public void registerParsingEnd() {
2112         parserCount_--;
2113     }
2114 
2115     /**
2116      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2117      *
2118      * Returns {@code true} if an HTML parser is parsing a non-inline HTML snippet to add content
2119      * to this page. Non-inline content is content that is parsed for the page, but not in the
2120      * same stream as the page itself -- basically anything other than <code>document.write()</code>
2121      * or <code>document.writeln()</code>: <code>innerHTML</code>, <code>outerHTML</code>,
2122      * <code>document.createElement()</code>, etc.
2123      *
2124      * @return {@code true} if an HTML parser is parsing a non-inline HTML snippet to add content
2125      *         to this page
2126      */
2127     public boolean isParsingHtmlSnippet() {
2128         return snippetParserCount_ > 0;
2129     }
2130 
2131     /**
2132      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2133      *
2134      * Called by the HTML parser to let the page know that it has started parsing a non-inline HTML snippet.
2135      */
2136     public void registerSnippetParsingStart() {
2137         snippetParserCount_++;
2138     }
2139 
2140     /**
2141      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2142      *
2143      * Called by the HTML parser to let the page know that it has finished parsing a non-inline HTML snippet.
2144      */
2145     public void registerSnippetParsingEnd() {
2146         snippetParserCount_--;
2147     }
2148 
2149     /**
2150      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2151      *
2152      * Returns {@code true} if an HTML parser is parsing an inline HTML snippet to add content
2153      * to this page. Inline content is content inserted into the parser stream dynamically
2154      * while the page is being parsed (i.e. <code>document.write()</code> or <code>document.writeln()</code>).
2155      *
2156      * @return {@code true} if an HTML parser is parsing an inline HTML snippet to add content
2157      *         to this page
2158      */
2159     public boolean isParsingInlineHtmlSnippet() {
2160         return inlineSnippetParserCount_ > 0;
2161     }
2162 
2163     /**
2164      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2165      *
2166      * Called by the HTML parser to let the page know that it has started parsing an inline HTML snippet.
2167      */
2168     public void registerInlineSnippetParsingStart() {
2169         inlineSnippetParserCount_++;
2170     }
2171 
2172     /**
2173      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2174      *
2175      * Called by the HTML parser to let the page know that it has finished parsing an inline HTML snippet.
2176      */
2177     public void registerInlineSnippetParsingEnd() {
2178         inlineSnippetParserCount_--;
2179     }
2180 
2181     /**
2182      * Refreshes the page by sending the same parameters as previously sent to get this page.
2183      * @return the newly loaded page.
2184      * @throws IOException if an IO problem occurs
2185      */
2186     public Page refresh() throws IOException {
2187         return getWebClient().getPage(getWebResponse().getWebRequest());
2188     }
2189 
2190     /**
2191      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2192      * <p>
2193      * Parses the given string as would it belong to the content being parsed
2194      * at the current parsing position
2195      * </p>
2196      * @param string the HTML code to write in place
2197      */
2198     public void writeInParsedStream(final String string) {
2199         getDOMBuilder().pushInputString(string);
2200     }
2201 
2202     /**
2203      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2204      *
2205      * Sets the builder to allow page to send content from document.write(ln) calls.
2206      * @param htmlUnitDOMBuilder the builder
2207      */
2208     public void setDOMBuilder(final HTMLParserDOMBuilder htmlUnitDOMBuilder) {
2209         domBuilder_ = htmlUnitDOMBuilder;
2210     }
2211 
2212     /**
2213      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2214      *
2215      * Returns the current builder.
2216      * @return the current builder
2217      */
2218     public HTMLParserDOMBuilder getDOMBuilder() {
2219         return domBuilder_;
2220     }
2221 
2222     /**
2223      * <p>Returns all namespaces defined in the root element of this page.</p>
2224      * <p>The default namespace has a key of an empty string.</p>
2225      * @return all namespaces defined in the root element of this page
2226      */
2227     public Map<String, String> getNamespaces() {
2228         final org.w3c.dom.NamedNodeMap attributes = getDocumentElement().getAttributes();
2229         final Map<String, String> namespaces = new HashMap<>();
2230         for (int i = 0; i < attributes.getLength(); i++) {
2231             final Attr attr = (Attr) attributes.item(i);
2232             String name = attr.getName();
2233             if (name.startsWith("xmlns")) {
2234                 int startPos = 5;
2235                 if (name.length() > 5 && name.charAt(5) == ':') {
2236                     startPos = 6;
2237                 }
2238                 name = name.substring(startPos);
2239                 namespaces.put(name, attr.getValue());
2240             }
2241         }
2242         return namespaces;
2243     }
2244 
2245     /**
2246      * {@inheritDoc}
2247      */
2248     @Override
2249     public void setDocumentType(final DocumentType type) {
2250         super.setDocumentType(type);
2251     }
2252 
2253     /**
2254      * Saves the current page, with all images, to the specified location.
2255      * The default behavior removes all script elements.
2256      *
2257      * @param file file to write this page into
2258      * @throws IOException If an error occurs
2259      */
2260     public void save(final File file) throws IOException {
2261         new XmlSerializer().save(this, file);
2262     }
2263 
2264     /**
2265      * Returns whether the current page mode is in {@code quirks mode} or in {@code standards mode}.
2266      * @return true for {@code quirks mode}, false for {@code standards mode}
2267      */
2268     public boolean isQuirksMode() {
2269         return "BackCompat".equals(((HTMLDocument) getScriptableObject()).getCompatMode());
2270     }
2271 
2272     /**
2273      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2274      * {@inheritDoc}
2275      */
2276     @Override
2277     public boolean isAttachedToPage() {
2278         return true;
2279     }
2280 
2281     /**
2282      * {@inheritDoc}
2283      */
2284     @Override
2285     public boolean isHtmlPage() {
2286         return true;
2287     }
2288 
2289     /**
2290      * The base URL used to resolve relative URLs.
2291      * @return the base URL
2292      */
2293     public URL getBaseURL() {
2294         URL baseUrl;
2295         if (base_ == null) {
2296             baseUrl = getUrl();
2297             final WebWindow window = getEnclosingWindow();
2298             final boolean frame = window != null && window != window.getTopWindow();
2299             if (frame) {
2300                 final boolean frameSrcIsNotSet = baseUrl == UrlUtils.URL_ABOUT_BLANK;
2301                 final boolean frameSrcIsJs = "javascript".equals(baseUrl.getProtocol());
2302                 if (frameSrcIsNotSet || frameSrcIsJs) {
2303                     baseUrl = window.getTopWindow().getEnclosedPage().getWebResponse()
2304                         .getWebRequest().getUrl();
2305                 }
2306             }
2307             else if (baseUrl_ != null) {
2308                 baseUrl = baseUrl_;
2309             }
2310         }
2311         else {
2312             final String href = base_.getHrefAttribute().trim();
2313             if (StringUtils.isEmpty(href)) {
2314                 baseUrl = getUrl();
2315             }
2316             else {
2317                 final URL url = getUrl();
2318                 try {
2319                     if (href.startsWith("http://") || href.startsWith("https://")) {
2320                         baseUrl = new URL(href);
2321                     }
2322                     else if (href.startsWith("//")) {
2323                         baseUrl = new URL(String.format("%s:%s", url.getProtocol(), href));
2324                     }
2325                     else if (href.length() > 0 && href.charAt(0) == '/') {
2326                         final int port = Window.getPort(url);
2327                         baseUrl = new URL(String.format("%s://%s:%d%s", url.getProtocol(), url.getHost(), port, href));
2328                     }
2329                     else if (url.toString().endsWith("/")) {
2330                         baseUrl = new URL(String.format("%s%s", url, href));
2331                     }
2332                     else {
2333                         baseUrl = new URL(UrlUtils.resolveUrl(url, href));
2334                     }
2335                 }
2336                 catch (final MalformedURLException e) {
2337                     notifyIncorrectness("Invalid base url: \"" + href + "\", ignoring it");
2338                     baseUrl = url;
2339                 }
2340             }
2341         }
2342 
2343         return baseUrl;
2344     }
2345 
2346     /**
2347      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2348      *
2349      * Adds an {@link AutoCloseable}, which would be closed during the {@link #cleanUp()}.
2350      * @param autoCloseable the autoclosable
2351      */
2352     public void addAutoCloseable(final AutoCloseable autoCloseable) {
2353         if (autoCloseable == null) {
2354             return;
2355         }
2356 
2357         if (autoCloseableList_ == null) {
2358             autoCloseableList_ = new ArrayList<>();
2359         }
2360         autoCloseableList_.add(autoCloseable);
2361     }
2362 
2363     /**
2364      * {@inheritDoc}
2365      */
2366     @Override
2367     public boolean handles(final Event event) {
2368         if (Event.TYPE_BLUR.equals(event.getType()) || Event.TYPE_FOCUS.equals(event.getType())) {
2369             return true;
2370         }
2371         return super.handles(event);
2372     }
2373 
2374     /**
2375      * Sets the {@link ElementFromPointHandler}.
2376      * @param elementFromPointHandler the handler
2377      */
2378     public void setElementFromPointHandler(final ElementFromPointHandler elementFromPointHandler) {
2379         elementFromPointHandler_ = elementFromPointHandler;
2380     }
2381 
2382     /**
2383      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2384      *
2385      * Returns the element for the specified x coordinate and the specified y coordinate.
2386      *
2387      * @param x the x offset, in pixels
2388      * @param y the y offset, in pixels
2389      * @return the element for the specified x coordinate and the specified y coordinate
2390      */
2391     public HtmlElement getElementFromPoint(final int x, final int y) {
2392         if (elementFromPointHandler_ == null) {
2393             if (LOG.isWarnEnabled()) {
2394                 LOG.warn("ElementFromPointHandler was not specicifed for " + this);
2395             }
2396             if (x <= 0 || y <= 0) {
2397                 return null;
2398             }
2399             return getBody();
2400         }
2401         return elementFromPointHandler_.getElementFromPoint(this, x, y);
2402     }
2403 
2404     /**
2405      * Moves the focus to the specified element. This will trigger any relevant JavaScript
2406      * event handlers.
2407      *
2408      * @param newElement the element that will receive the focus, use {@code null} to remove focus from any element
2409      * @return true if the specified element now has the focus
2410      * @see #getFocusedElement()
2411      */
2412     public boolean setFocusedElement(final DomElement newElement) {
2413         return setFocusedElement(newElement, false);
2414     }
2415 
2416     /**
2417      * Moves the focus to the specified element. This will trigger any relevant JavaScript
2418      * event handlers.
2419      *
2420      * @param newElement the element that will receive the focus, use {@code null} to remove focus from any element
2421      * @param windowActivated - whether the enclosing window got focus resulting in specified element getting focus
2422      * @return true if the specified element now has the focus
2423      * @see #getFocusedElement()
2424      */
2425     public boolean setFocusedElement(final DomElement newElement, final boolean windowActivated) {
2426         if (elementWithFocus_ == newElement && !windowActivated) {
2427             // nothing to do
2428             return true;
2429         }
2430 
2431         final DomElement oldFocusedElement = elementWithFocus_;
2432         elementWithFocus_ = null;
2433 
2434         if (!windowActivated) {
2435             if (oldFocusedElement != null) {
2436                 oldFocusedElement.removeFocus();
2437                 oldFocusedElement.fireEvent(Event.TYPE_BLUR);
2438 
2439                 oldFocusedElement.fireEvent(Event.TYPE_FOCUS_OUT);
2440             }
2441         }
2442 
2443         elementWithFocus_ = newElement;
2444 
2445         // use newElement in the code below because element elementWithFocus_
2446         // might be changed by another thread
2447         if (newElement != null) {
2448             newElement.focus();
2449             newElement.fireEvent(Event.TYPE_FOCUS);
2450 
2451             newElement.fireEvent(Event.TYPE_FOCUS_IN);
2452         }
2453 
2454         // If a page reload happened as a result of the focus change then obviously this
2455         // element will not have the focus because its page has gone away.
2456         return this == getEnclosingWindow().getEnclosedPage();
2457     }
2458 
2459     /**
2460      * Returns the element with the focus or null if no element has the focus.
2461      * @return the element with focus or null
2462      * @see #setFocusedElement(DomElement)
2463      */
2464     public DomElement getFocusedElement() {
2465         return elementWithFocus_;
2466     }
2467 
2468     /**
2469      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2470      *
2471      * Sets the element with focus.
2472      * @param elementWithFocus the element with focus
2473      */
2474     public void setElementWithFocus(final DomElement elementWithFocus) {
2475         elementWithFocus_ = elementWithFocus;
2476     }
2477 
2478     /**
2479      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2480      *
2481      * @return the element with focus or the body
2482      */
2483     public HtmlElement getActiveElement() {
2484         final DomElement activeElement = getFocusedElement();
2485         if (activeElement instanceof HtmlElement) {
2486             return (HtmlElement) activeElement;
2487         }
2488 
2489         final HtmlElement body = getBody();
2490         if (body != null) {
2491             return body;
2492         }
2493         return null;
2494     }
2495 
2496     /**
2497      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2498      *
2499      * <p>Returns the page's current selection ranges.</p>
2500      *
2501      * @return the page's current selection ranges
2502      */
2503     public List<SimpleRange> getSelectionRanges() {
2504         return selectionRanges_;
2505     }
2506 
2507     /**
2508      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2509      *
2510      * <p>Makes the specified selection range the *only* selection range on this page.</p>
2511      *
2512      * @param selectionRange the selection range
2513      */
2514     public void setSelectionRange(final SimpleRange selectionRange) {
2515         selectionRanges_.clear();
2516         selectionRanges_.add(selectionRange);
2517     }
2518 
2519     /**
2520      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2521      *
2522      * Execute a Function in the given context.
2523      *
2524      * @param function the JavaScript Function to call
2525      * @param thisObject the "this" object to be used during invocation
2526      * @param args the arguments to pass into the call
2527      * @param htmlElementScope the HTML element for which this script is being executed
2528      *        This element will be the context during the JavaScript execution. If null,
2529      *        the context will default to the page.
2530      * @return a ScriptResult which will contain both the current page (which may be different than
2531      *        the previous page and a JavaScript result object.
2532      */
2533     public ScriptResult executeJavaScriptFunction(final Object function, final Object thisObject,
2534             final Object[] args, final DomNode htmlElementScope) {
2535         if (!getWebClient().isJavaScriptEnabled()) {
2536             return new ScriptResult(null);
2537         }
2538 
2539         return executeJavaScriptFunction((Function) function, (Scriptable) thisObject, args, htmlElementScope);
2540     }
2541 
2542     private ScriptResult executeJavaScriptFunction(final Function function, final Scriptable thisObject,
2543             final Object[] args, final DomNode htmlElementScope) {
2544 
2545         final JavaScriptEngine engine = (JavaScriptEngine) getWebClient().getJavaScriptEngine();
2546         final Object result = engine.callFunction(this, function, thisObject, args, htmlElementScope);
2547 
2548         return new ScriptResult(result);
2549     }
2550 
2551     private void writeObject(final ObjectOutputStream oos) throws IOException {
2552         oos.defaultWriteObject();
2553         oos.writeObject(originalCharset_ == null ? null : originalCharset_.name());
2554     }
2555 
2556     private void readObject(final ObjectInputStream ois) throws ClassNotFoundException, IOException {
2557         ois.defaultReadObject();
2558         final String charsetName = (String) ois.readObject();
2559         if (charsetName != null) {
2560             originalCharset_ = Charset.forName(charsetName);
2561         }
2562     }
2563 
2564     /**
2565      * {@inheritDoc}
2566      */
2567     @Override
2568     public void setNodeValue(final String value) {
2569         // Default behavior is to do nothing, overridden in some subclasses
2570     }
2571 
2572     /**
2573      * {@inheritDoc}
2574      */
2575     @Override
2576     public void setPrefix(final String prefix) {
2577         // Empty.
2578     }
2579 
2580     /**
2581      * {@inheritDoc}
2582      */
2583     @Override
2584     public void clearComputedStyles() {
2585         if (computedStylesCache_ != null) {
2586             computedStylesCache_.clear();
2587         }
2588     }
2589 
2590     /**
2591      * {@inheritDoc}
2592      */
2593     @Override
2594     public void clearComputedStyles(final DomElement element) {
2595         if (computedStylesCache_ != null) {
2596             computedStylesCache_.remove(element);
2597         }
2598     }
2599 
2600     /**
2601      * {@inheritDoc}
2602      */
2603     @Override
2604     public void clearComputedStylesUpToRoot(final DomElement element) {
2605         if (computedStylesCache_ != null) {
2606             computedStylesCache_.remove(element);
2607 
2608             DomNode parent = element.getParentNode();
2609             while (parent != null) {
2610                 computedStylesCache_.remove(parent);
2611                 parent = parent.getParentNode();
2612             }
2613         }
2614     }
2615 
2616     /**
2617      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2618      *
2619      * @param element the element to clear its cache
2620      * @param normalizedPseudo the pseudo attribute
2621      * @return the cached CSS2Properties object or null
2622      */
2623     public ComputedCssStyleDeclaration getStyleFromCache(final DomElement element,
2624             final String normalizedPseudo) {
2625         return getCssPropertiesCache().get(element, normalizedPseudo);
2626     }
2627 
2628     /**
2629      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2630      *
2631      * Caches a CSS2Properties object.
2632      * @param element the element to clear its cache
2633      * @param normalizedPseudo the pseudo attribute
2634      * @param style the CSS2Properties to cache
2635      */
2636     public void putStyleIntoCache(final DomElement element, final String normalizedPseudo,
2637             final ComputedCssStyleDeclaration style) {
2638         getCssPropertiesCache().put(element, normalizedPseudo, style);
2639     }
2640 
2641     /**
2642      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2643      *
2644      * @return a list of all styles from this page (&lt;style&gt; and &lt;link rel=stylesheet&gt;).
2645      *         This returns an empty list if css support is disabled in the web client options.
2646      */
2647     public List<CssStyleSheet> getStyleSheets() {
2648         final List<CssStyleSheet> styles = new ArrayList<>();
2649         if (getWebClient().getOptions().isCssEnabled()) {
2650             for (final HtmlElement htmlElement : getHtmlElementDescendants()) {
2651                 if (htmlElement instanceof HtmlStyle) {
2652                     styles.add(((HtmlStyle) htmlElement).getSheet());
2653                     continue;
2654                 }
2655 
2656                 if (htmlElement instanceof HtmlLink) {
2657                     final HtmlLink link = (HtmlLink) htmlElement;
2658                     if (link.isStyleSheetLink()) {
2659                         styles.add(link.getSheet());
2660                     }
2661                 }
2662             }
2663         }
2664         return styles;
2665     }
2666 
2667     /**
2668      * @return the CSSPropertiesCache for this page
2669      */
2670     private ComputedStylesCache getCssPropertiesCache() {
2671         if (computedStylesCache_ == null) {
2672             computedStylesCache_ = new ComputedStylesCache();
2673 
2674             // maintain the style cache
2675             final DomHtmlAttributeChangeListenerImpl listener = new DomHtmlAttributeChangeListenerImpl();
2676             addDomChangeListener(listener);
2677             addHtmlAttributeChangeListener(listener);
2678         }
2679         return computedStylesCache_;
2680     }
2681 
2682     /**
2683      * <p>Listens for changes anywhere in the document and evicts cached computed styles whenever something relevant
2684      * changes. Note that the very lazy way of doing this (completely clearing the cache every time something happens)
2685      * results in very meager performance gains. In order to get good (but still correct) performance, we need to be
2686      * a little smarter.</p>
2687      *
2688      * <p>CSS 2.1 has the following <a href="http://www.w3.org/TR/CSS21/selector.html">selector types</a> (where "SN" is
2689      * shorthand for "the selected node"):</p>
2690      *
2691      * <ol>
2692      *   <li><em>Universal</em> (i.e. "*"): Affected by the removal of SN from the document.</li>
2693      *   <li><em>Type</em> (i.e. "div"): Affected by the removal of SN from the document.</li>
2694      *   <li><em>Descendant</em> (i.e. "div span"): Affected by changes to SN or to any of its ancestors.</li>
2695      *   <li><em>Child</em> (i.e. "div &gt; span"): Affected by changes to SN or to its parent.</li>
2696      *   <li><em>Adjacent Sibling</em> (i.e. "table + p"): Affected by changes to SN or its previous sibling.</li>
2697      *   <li><em>Attribute</em> (i.e. "div.up, div[class~=up]"): Affected by changes to an attribute of SN.</li>
2698      *   <li><em>ID</em> (i.e. "#header): Affected by changes to the <code>id</code> attribute of SN.</li>
2699      *   <li><em>Pseudo-Elements and Pseudo-Classes</em> (i.e. "p:first-child"): Affected by changes to parent.</li>
2700      * </ol>
2701      *
2702      * <p>Together, these rules dictate that the smart (but still lazy) way of removing elements from the computed style
2703      * cache is as follows -- whenever a node changes in any way, the cache needs to be cleared of styles for nodes
2704      * which:</p>
2705      *
2706      * <ul>
2707      *   <li>are actually the same node as the node that changed</li>
2708      *   <li>are siblings of the node that changed</li>
2709      *   <li>are descendants of the node that changed</li>
2710      * </ul>
2711      *
2712      * <p>Additionally, whenever a <code>style</code> node or a <code>link</code> node
2713      * with <code>rel=stylesheet</code> is added or
2714      * removed, all elements should be removed from the computed style cache.</p>
2715      */
2716     private class DomHtmlAttributeChangeListenerImpl implements DomChangeListener, HtmlAttributeChangeListener {
2717 
2718         /**
2719          * Ctor.
2720          */
2721         DomHtmlAttributeChangeListenerImpl() {
2722             super();
2723         }
2724 
2725         /**
2726          * {@inheritDoc}
2727          */
2728         @Override
2729         public void nodeAdded(final DomChangeEvent event) {
2730             nodeChanged(event.getChangedNode(), null);
2731         }
2732 
2733         /**
2734          * {@inheritDoc}
2735          */
2736         @Override
2737         public void nodeDeleted(final DomChangeEvent event) {
2738             nodeChanged(event.getChangedNode(), null);
2739         }
2740 
2741         /**
2742          * {@inheritDoc}
2743          */
2744         @Override
2745         public void attributeAdded(final HtmlAttributeChangeEvent event) {
2746             nodeChanged(event.getHtmlElement(), event.getName());
2747         }
2748 
2749         /**
2750          * {@inheritDoc}
2751          */
2752         @Override
2753         public void attributeRemoved(final HtmlAttributeChangeEvent event) {
2754             nodeChanged(event.getHtmlElement(), event.getName());
2755         }
2756 
2757         /**
2758          * {@inheritDoc}
2759          */
2760         @Override
2761         public void attributeReplaced(final HtmlAttributeChangeEvent event) {
2762             nodeChanged(event.getHtmlElement(), event.getName());
2763         }
2764 
2765         private void nodeChanged(final DomNode changedNode, final String attribName) {
2766             // If a stylesheet was changed, all of our calculations could be off; clear the cache.
2767             if (changedNode instanceof HtmlStyle) {
2768                 clearComputedStyles();
2769                 return;
2770             }
2771             if (changedNode instanceof HtmlLink) {
2772                 if (((HtmlLink) changedNode).isStyleSheetLink()) {
2773                     clearComputedStyles();
2774                     return;
2775                 }
2776             }
2777 
2778             // Apparently it wasn't a stylesheet that changed; be semi-smart about what we evict and when.
2779             // null means that a node was added/removed; we always have to take care of this for the parents
2780             final boolean clearParents = attribName == null || ATTRIBUTES_AFFECTING_PARENT.contains(attribName);
2781             if (computedStylesCache_ != null) {
2782                 computedStylesCache_.nodeChanged(changedNode, clearParents);
2783             }
2784         }
2785     }
2786 
2787     /**
2788      * Cache computed styles when possible, because their calculation is very expensive.
2789      * We use a weak hash map because we don't want this cache to be the only reason
2790      * nodes are kept around in the JVM, if all other references to them are gone.
2791      */
2792     private static final class ComputedStylesCache implements Serializable {
2793         private transient WeakHashMap<DomElement, Map<String, ComputedCssStyleDeclaration>>
2794                     computedStyles_ = new WeakHashMap<>();
2795 
2796         /**
2797          * Ctor.
2798          */
2799         ComputedStylesCache() {
2800             super();
2801         }
2802 
2803         public synchronized ComputedCssStyleDeclaration get(final DomElement element,
2804                 final String normalizedPseudo) {
2805             final Map<String, ComputedCssStyleDeclaration> elementMap = computedStyles_.get(element);
2806             if (elementMap != null) {
2807                 return elementMap.get(normalizedPseudo);
2808             }
2809             return null;
2810         }
2811 
2812         public synchronized void put(final DomElement element,
2813                 final String normalizedPseudo, final ComputedCssStyleDeclaration style) {
2814             final Map<String, ComputedCssStyleDeclaration>
2815                     elementMap = computedStyles_.computeIfAbsent(element, k -> new WeakHashMap<>());
2816             elementMap.put(normalizedPseudo, style);
2817         }
2818 
2819         public synchronized void nodeChanged(final DomNode changed, final boolean clearParents) {
2820             final Iterator<Map.Entry<DomElement, Map<String, ComputedCssStyleDeclaration>>>
2821                     i = computedStyles_.entrySet().iterator();
2822             while (i.hasNext()) {
2823                 final Map.Entry<DomElement, Map<String, ComputedCssStyleDeclaration>> entry = i.next();
2824                 final DomElement node = entry.getKey();
2825                 if (changed == node
2826                     || changed.getParentNode() == node.getParentNode()
2827                     || changed.isAncestorOf(node)
2828                     || clearParents && node.isAncestorOf(changed)) {
2829                     i.remove();
2830                 }
2831             }
2832 
2833             // maybe this is a better solution but i have to think a bit more about this
2834             //
2835             //            if (computedStyles_.isEmpty()) {
2836             //                return;
2837             //            }
2838             //
2839             //            // remove all siblings
2840             //            DomNode parent = changed.getParentNode();
2841             //            if (parent != null) {
2842             //                for (DomNode sibling : parent.getChildNodes()) {
2843             //                    computedStyles_.remove(sibling.getScriptableObject());
2844             //                }
2845             //
2846             //                if (clearParents) {
2847             //                    // remove all parents
2848             //                    while (parent != null) {
2849             //                        computedStyles_.remove(parent.getScriptableObject());
2850             //                        parent = parent.getParentNode();
2851             //                    }
2852             //                }
2853             //            }
2854             //
2855             //            // remove changed itself and all descendants
2856             //            computedStyles_.remove(changed.getScriptableObject());
2857             //            for (DomNode descendant : changed.getDescendants()) {
2858             //                computedStyles_.remove(descendant.getScriptableObject());
2859             //            }
2860         }
2861 
2862         public synchronized void clear() {
2863             computedStyles_.clear();
2864         }
2865 
2866         public synchronized Map<String, ComputedCssStyleDeclaration> remove(
2867                 final DomNode element) {
2868             return computedStyles_.remove(element);
2869         }
2870 
2871         private void readObject(final ObjectInputStream in) throws IOException, ClassNotFoundException {
2872             in.defaultReadObject();
2873             computedStyles_ = new WeakHashMap<>();
2874         }
2875     }
2876 
2877     private static final class MappedElementIndexEntry implements Serializable {
2878         private final ArrayList<DomElement> elements_;
2879         private boolean sorted_;
2880 
2881         MappedElementIndexEntry() {
2882             // we do not expect to many elements having the same id/name
2883             elements_ = new ArrayList<>(2);
2884             sorted_ = false;
2885         }
2886 
2887         void add(final DomElement element) {
2888             elements_.add(element);
2889             sorted_ = false;
2890         }
2891 
2892         DomElement first() {
2893             if (elements_.size() == 0) {
2894                 return null;
2895             }
2896 
2897             if (sorted_) {
2898                 return elements_.get(0);
2899             }
2900 
2901             Collections.sort(elements_, DOCUMENT_POSITION_COMPERATOR);
2902             sorted_ = true;
2903 
2904             return elements_.get(0);
2905         }
2906 
2907         List<DomElement> elements() {
2908             if (sorted_ || elements_.size() == 0) {
2909                 return elements_;
2910             }
2911 
2912             Collections.sort(elements_, DOCUMENT_POSITION_COMPERATOR);
2913             sorted_ = true;
2914 
2915             return elements_;
2916         }
2917 
2918         boolean remove(final DomElement element) {
2919             if (elements_.size() == 0) {
2920                 return false;
2921             }
2922 
2923             return elements_.remove(element);
2924         }
2925     }
2926 }