View Javadoc
1   /*
2    * Copyright (c) 2002-2026 Gargoyle Software Inc.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    * https://www.apache.org/licenses/LICENSE-2.0
8    *
9    * Unless required by applicable law or agreed to in writing, software
10   * distributed under the License is distributed on an "AS IS" BASIS,
11   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12   * See the License for the specific language governing permissions and
13   * limitations under the License.
14   */
15  package org.htmlunit.html;
16  
17  import static org.htmlunit.BrowserVersionFeatures.EVENT_FOCUS_ON_LOAD;
18  import static org.htmlunit.BrowserVersionFeatures.HTTP_HEADER_CH_UA;
19  import static org.htmlunit.html.DomElement.ATTRIBUTE_NOT_DEFINED;
20  
21  import java.io.File;
22  import java.io.IOException;
23  import java.io.ObjectInputStream;
24  import java.io.ObjectOutputStream;
25  import java.io.Serializable;
26  import java.net.MalformedURLException;
27  import java.net.URL;
28  import java.nio.charset.Charset;
29  import java.nio.charset.StandardCharsets;
30  import java.util.ArrayList;
31  import java.util.Arrays;
32  import java.util.Collection;
33  import java.util.Collections;
34  import java.util.Comparator;
35  import java.util.HashMap;
36  import java.util.HashSet;
37  import java.util.Iterator;
38  import java.util.LinkedHashSet;
39  import java.util.List;
40  import java.util.Locale;
41  import java.util.Map;
42  import java.util.Set;
43  import java.util.WeakHashMap;
44  import java.util.concurrent.ConcurrentHashMap;
45  
46  import javax.xml.XMLConstants;
47  
48  import org.apache.commons.lang3.StringUtils;
49  import org.apache.commons.logging.Log;
50  import org.apache.commons.logging.LogFactory;
51  import org.htmlunit.Cache;
52  import org.htmlunit.ElementNotFoundException;
53  import org.htmlunit.FailingHttpStatusCodeException;
54  import org.htmlunit.History;
55  import org.htmlunit.HttpHeader;
56  import org.htmlunit.OnbeforeunloadHandler;
57  import org.htmlunit.Page;
58  import org.htmlunit.ScriptResult;
59  import org.htmlunit.SgmlPage;
60  import org.htmlunit.TopLevelWindow;
61  import org.htmlunit.WebAssert;
62  import org.htmlunit.WebClient;
63  import org.htmlunit.WebClientOptions;
64  import org.htmlunit.WebRequest;
65  import org.htmlunit.WebResponse;
66  import org.htmlunit.WebWindow;
67  import org.htmlunit.corejs.javascript.Function;
68  import org.htmlunit.corejs.javascript.Script;
69  import org.htmlunit.corejs.javascript.Scriptable;
70  import org.htmlunit.corejs.javascript.ScriptableObject;
71  import org.htmlunit.corejs.javascript.VarScope;
72  import org.htmlunit.css.ComputedCssStyleDeclaration;
73  import org.htmlunit.css.CssStyleSheet;
74  import org.htmlunit.html.impl.SimpleRange;
75  import org.htmlunit.html.parser.HTMLParserDOMBuilder;
76  import org.htmlunit.http.HttpStatus;
77  import org.htmlunit.javascript.AbstractJavaScriptEngine;
78  import org.htmlunit.javascript.HtmlUnitScriptable;
79  import org.htmlunit.javascript.JavaScriptEngine;
80  import org.htmlunit.javascript.PostponedAction;
81  import org.htmlunit.javascript.host.Window;
82  import org.htmlunit.javascript.host.event.BeforeUnloadEvent;
83  import org.htmlunit.javascript.host.event.Event;
84  import org.htmlunit.javascript.host.event.EventTarget;
85  import org.htmlunit.javascript.host.html.HTMLDocument;
86  import org.htmlunit.protocol.javascript.JavaScriptURLConnection;
87  import org.htmlunit.util.MimeType;
88  import org.htmlunit.util.SerializableLock;
89  import org.htmlunit.util.UrlUtils;
90  import org.w3c.dom.Attr;
91  import org.w3c.dom.Comment;
92  import org.w3c.dom.DOMConfiguration;
93  import org.w3c.dom.DOMException;
94  import org.w3c.dom.DOMImplementation;
95  import org.w3c.dom.Document;
96  import org.w3c.dom.DocumentType;
97  import org.w3c.dom.Element;
98  import org.w3c.dom.EntityReference;
99  import org.w3c.dom.ProcessingInstruction;
100 
101 /**
102  * A representation of an HTML page returned from a server.
103  * <p>
104  * This class provides different methods to access the page's content like
105  * {@link #getForms()}, {@link #getAnchors()}, {@link #getElementById(String)}, ... as well as the
106  * very powerful inherited methods {@link #getByXPath(String)} and {@link #getFirstByXPath(String)}
107  * for fine grained user specific access to child nodes.
108  * </p>
109  * <p>
110  * Child elements allowing user interaction provide methods for this purpose like {@link HtmlAnchor#click()},
111  * {@link HtmlInput#type(String)}, {@link HtmlOption#setSelected(boolean)}, ...
112  * </p>
113  * <p>
114  * HtmlPage instances should not be instantiated directly. They will be returned by {@link WebClient#getPage(String)}
115  * when the content type of the server's response is <code>text/html</code> (or one of its variations).<br>
116  * <br>
117  * <b>Example:</b><br>
118  * <br>
119  * <code>
120  * final HtmlPage page = webClient.{@link WebClient#getPage(String) getPage}("http://mywebsite/some/page.html");
121  * </code>
122  * </p>
123  *
124  * @author Mike Bowler
125  * @author Alex Nikiforoff
126  * @author Noboru Sinohara
127  * @author David K. Taylor
128  * @author Andreas Hangler
129  * @author Christian Sell
130  * @author Chris Erskine
131  * @author Marc Guillemot
132  * @author Ahmed Ashour
133  * @author Daniel Gredler
134  * @author Dmitri Zoubkov
135  * @author Sudhan Moghe
136  * @author Ethan Glasser-Camp
137  * @author Tom Anderson
138  * @author Ronald Brill
139  * @author Frank Danek
140  * @author Joerg Werner
141  * @author Atsushi Nakagawa
142  * @author Rural Hunter
143  * @author Ronny Shapiro
144  * @author Lai Quang Duong
145  * @author Sven Strickroth
146  * @author Vidya m
147  */
148 @SuppressWarnings("PMD.TooManyFields")
149 public class HtmlPage extends SgmlPage {
150 
151     private static final Log LOG = LogFactory.getLog(HtmlPage.class);
152 
153     private static final Comparator<DomElement> DOCUMENT_POSITION_COMPERATOR = new DocumentPositionComparator();
154 
155     private HTMLParserDOMBuilder domBuilder_;
156     private transient Charset originalCharset_;
157     private final Object lock_ = new SerializableLock(); // used for synchronization
158 
159     private Map<String, MappedElementIndexEntry> idMap_ = new ConcurrentHashMap<>();
160     private Map<String, MappedElementIndexEntry> nameMap_ = new ConcurrentHashMap<>();
161     // The id/name lookup index is built lazily on first use. Until then,
162     // notifyNodeAdded / fireAttributeChange skip the per-element index updates.
163     // Reads must call ensureMappedElementsBuilt() before consulting idMap_/nameMap_.
164     private boolean mappedElementsBuilt_;
165 
166     private List<BaseFrameElement> frameElements_ = new ArrayList<>();
167     private int parserCount_;
168     private int snippetParserCount_;
169     private int inlineSnippetParserCount_;
170     private Collection<HtmlAttributeChangeListener> attributeListeners_;
171     private List<PostponedAction> afterLoadActions_ = Collections.synchronizedList(new ArrayList<>());
172     private boolean cleaning_;
173     private HtmlBase base_;
174     private URL baseUrl_;
175     private List<AutoCloseable> autoCloseableList_;
176     private ElementFromPointHandler elementFromPointHandler_;
177     private DomElement elementWithFocus_;
178     private List<SimpleRange> selectionRanges_ = new ArrayList<>(3);
179 
180     private transient ComputedStylesCache computedStylesCache_;
181 
182     private static final HashSet<String> TABBABLE_TAGS =
183             new HashSet<>(Arrays.asList(HtmlAnchor.TAG_NAME, HtmlArea.TAG_NAME,
184                     HtmlButton.TAG_NAME, HtmlInput.TAG_NAME, HtmlObject.TAG_NAME,
185                     HtmlSelect.TAG_NAME, HtmlTextArea.TAG_NAME));
186     private static final HashSet<String> ACCEPTABLE_TAG_NAMES =
187             new HashSet<>(Arrays.asList(HtmlAnchor.TAG_NAME, HtmlArea.TAG_NAME,
188                     HtmlButton.TAG_NAME, HtmlInput.TAG_NAME, HtmlLabel.TAG_NAME,
189                     HtmlLegend.TAG_NAME, HtmlTextArea.TAG_NAME));
190 
191     /** Definition of special cases for the smart DomHtmlAttributeChangeListenerImpl. */
192     private static final Set<String> ATTRIBUTES_AFFECTING_PARENT = new HashSet<>(Arrays.asList(
193             "style",
194             "class",
195             "height",
196             "width"));
197 
198     static class DocumentPositionComparator implements Comparator<DomElement>, Serializable {
199         @Override
200         public int compare(final DomElement elt1, final DomElement elt2) {
201             final short relation = elt1.compareDocumentPosition(elt2);
202             if (relation == 0) {
203                 return 0; // same node
204             }
205             if ((relation & DOCUMENT_POSITION_CONTAINS) != 0 || (relation & DOCUMENT_POSITION_PRECEDING) != 0) {
206                 return 1;
207             }
208 
209             return -1;
210         }
211     }
212 
213     /**
214      * Creates an instance of HtmlPage.
215      * An HtmlPage instance is normally retrieved with {@link WebClient#getPage(String)}.
216      *
217      * @param webResponse the web response that was used to create this page
218      * @param webWindow the window that this page is being loaded into
219      */
220     public HtmlPage(final WebResponse webResponse, final WebWindow webWindow) {
221         super(webResponse, webWindow);
222     }
223 
224     /**
225      * {@inheritDoc}
226      */
227     @Override
228     public HtmlPage getPage() {
229         return this;
230     }
231 
232     /**
233      * {@inheritDoc}
234      */
235     @Override
236     public boolean hasCaseSensitiveTagNames() {
237         return false;
238     }
239 
240     /**
241      * Initialize this page.
242      * @throws IOException if an IO problem occurs
243      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
244      *         {@link org.htmlunit.WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set
245      *         to true.
246      */
247     @Override
248     public void initialize() throws IOException, FailingHttpStatusCodeException {
249         final WebWindow enclosingWindow = getEnclosingWindow();
250         final boolean isAboutBlank = getUrl() == UrlUtils.URL_ABOUT_BLANK;
251         if (isAboutBlank) {
252             // a frame contains first a faked "about:blank" before its real content specified by src gets loaded
253             if (enclosingWindow instanceof FrameWindow window
254                     && !window.getFrameElement().isContentLoaded()) {
255                 return;
256             }
257 
258             // save the URL that should be used to resolve relative URLs in this page
259             if (enclosingWindow instanceof TopLevelWindow topWindow) {
260                 final WebWindow openerWindow = topWindow.getOpener();
261                 if (openerWindow != null && openerWindow.getEnclosedPage() != null) {
262                     baseUrl_ = openerWindow.getEnclosedPage().getWebResponse().getWebRequest().getUrl();
263                 }
264             }
265         }
266 
267         if (!isAboutBlank) {
268             setReadyState(READY_STATE_INTERACTIVE);
269             getDocumentElement().setReadyState(READY_STATE_INTERACTIVE);
270             executeEventHandlersIfNeeded(Event.TYPE_READY_STATE_CHANGE);
271         }
272 
273         executeDeferredScriptsIfNeeded();
274 
275         executeEventHandlersIfNeeded(Event.TYPE_DOM_DOCUMENT_LOADED);
276 
277         // postponed actions are more or less the async scripts,
278         // they are running in real browsers whenever the download is done
279         processPostponedActionsIfNeeded();
280 
281         loadFrames();
282 
283         // don't set the ready state if we really load the blank page into the window
284         // see Node.initInlineFrameIfNeeded()
285         if (!isAboutBlank) {
286             setReadyState(READY_STATE_COMPLETE);
287             getDocumentElement().setReadyState(READY_STATE_COMPLETE);
288             executeEventHandlersIfNeeded(Event.TYPE_READY_STATE_CHANGE);
289         }
290 
291         // frame initialization has a different order
292         boolean isFrameWindow = enclosingWindow instanceof FrameWindow;
293         boolean isFirstPageInFrameWindow = false;
294         if (isFrameWindow) {
295             isFrameWindow = ((FrameWindow) enclosingWindow).getFrameElement() instanceof HtmlFrame;
296 
297             final History hist = enclosingWindow.getHistory();
298             if (hist.getLength() > 0 && UrlUtils.URL_ABOUT_BLANK == hist.getUrl(0)) {
299                 isFirstPageInFrameWindow = hist.getLength() <= 2;
300             }
301             else {
302                 isFirstPageInFrameWindow = enclosingWindow.getHistory().getLength() < 2;
303             }
304         }
305 
306         if (isFrameWindow && !isFirstPageInFrameWindow) {
307             executeEventHandlersIfNeeded(Event.TYPE_LOAD);
308         }
309 
310         for (final BaseFrameElement frameElement : new ArrayList<>(frameElements_)) {
311             if (frameElement instanceof HtmlFrame) {
312                 final Page page = frameElement.getEnclosedWindow().getEnclosedPage();
313                 if (page != null && page.isHtmlPage()) {
314                     ((HtmlPage) page).executeEventHandlersIfNeeded(Event.TYPE_LOAD);
315                 }
316             }
317         }
318 
319         if (!isFrameWindow) {
320             executeEventHandlersIfNeeded(Event.TYPE_LOAD);
321 
322             if (!isAboutBlank && enclosingWindow.getWebClient().isJavaScriptEnabled()
323                     && hasFeature(EVENT_FOCUS_ON_LOAD)) {
324                 final HtmlElement body = getBody();
325                 if (body != null) {
326                     final Event event = new Event((Window) enclosingWindow.getScriptableObject(), Event.TYPE_FOCUS);
327                     body.fireEvent(event);
328                 }
329             }
330         }
331 
332         try {
333             while (!afterLoadActions_.isEmpty()) {
334                 final PostponedAction action = afterLoadActions_.remove(0);
335                 action.execute();
336             }
337         }
338         catch (final IOException e) {
339             throw e;
340         }
341         catch (final Exception e) {
342             throw new RuntimeException(e);
343         }
344         executeRefreshIfNeeded();
345     }
346 
347     /**
348      * Adds an action that should be executed once the page has been loaded.
349      * @param action the action
350      */
351     void addAfterLoadAction(final PostponedAction action) {
352         afterLoadActions_.add(action);
353     }
354 
355     /**
356      * Clean up this page.
357      */
358     @Override
359     public void cleanUp() {
360         //To avoid endless recursion caused by window.close() in onUnload
361         if (cleaning_) {
362             return;
363         }
364 
365         cleaning_ = true;
366         try {
367             super.cleanUp();
368             executeEventHandlersIfNeeded(Event.TYPE_UNLOAD);
369             deregisterFramesIfNeeded();
370         }
371         finally {
372             cleaning_ = false;
373 
374             if (autoCloseableList_ != null) {
375                 for (final AutoCloseable closeable : new ArrayList<>(autoCloseableList_)) {
376                     try {
377                         closeable.close();
378                     }
379                     catch (final Exception e) {
380                         LOG.error("Closing the autoclosable " + closeable + " failed", e);
381                     }
382                 }
383             }
384         }
385     }
386 
387     /**
388      * {@inheritDoc}
389      */
390     @Override
391     public HtmlElement getDocumentElement() {
392         return (HtmlElement) super.getDocumentElement();
393     }
394 
395     /**
396      * Returns the document's {@code body} element.
397      *
398      * @return the document's {@code body} element, or {@code null} if it does
399      *         not exist
400      */
401     public HtmlBody getBody() {
402         final DomElement doc = getDocumentElement();
403         if (doc != null) {
404             for (final DomNode node : doc.getChildren()) {
405                 if (node instanceof HtmlBody body) {
406                     return body;
407                 }
408             }
409         }
410         return null;
411     }
412 
413     /**
414      * Returns the head element.
415      * @return the head element
416      */
417     public HtmlElement getHead() {
418         final DomElement doc = getDocumentElement();
419         if (doc != null) {
420             for (final DomNode node : doc.getChildren()) {
421                 if (node instanceof HtmlHead) {
422                     return (HtmlElement) node;
423                 }
424             }
425         }
426         return null;
427     }
428 
429     /**
430      * {@inheritDoc}
431      */
432     @Override
433     public Document getOwnerDocument() {
434         return null;
435     }
436 
437     /**
438      * {@inheritDoc}
439      * Not yet implemented.
440      */
441     @Override
442     public org.w3c.dom.Node importNode(final org.w3c.dom.Node importedNode, final boolean deep) {
443         throw new UnsupportedOperationException("HtmlPage.importNode is not yet implemented.");
444     }
445 
446     /**
447      * {@inheritDoc}
448      * Not yet implemented.
449      */
450     @Override
451     public String getInputEncoding() {
452         throw new UnsupportedOperationException("HtmlPage.getInputEncoding is not yet implemented.");
453     }
454 
455     /**
456      * {@inheritDoc}
457      */
458     @Override
459     public String getXmlEncoding() {
460         return null;
461     }
462 
463     /**
464      * {@inheritDoc}
465      */
466     @Override
467     public boolean getXmlStandalone() {
468         return false;
469     }
470 
471     /**
472      * {@inheritDoc}
473      * Not yet implemented.
474      */
475     @Override
476     public void setXmlStandalone(final boolean xmlStandalone) throws DOMException {
477         throw new UnsupportedOperationException("HtmlPage.setXmlStandalone is not yet implemented.");
478     }
479 
480     /**
481      * {@inheritDoc}
482      */
483     @Override
484     public String getXmlVersion() {
485         return null;
486     }
487 
488     /**
489      * {@inheritDoc}
490      * Not yet implemented.
491      */
492     @Override
493     public void setXmlVersion(final String xmlVersion) throws DOMException {
494         throw new UnsupportedOperationException("HtmlPage.setXmlVersion is not yet implemented.");
495     }
496 
497     /**
498      * {@inheritDoc}
499      * Not yet implemented.
500      */
501     @Override
502     public boolean getStrictErrorChecking() {
503         throw new UnsupportedOperationException("HtmlPage.getStrictErrorChecking is not yet implemented.");
504     }
505 
506     /**
507      * {@inheritDoc}
508      * Not yet implemented.
509      */
510     @Override
511     public void setStrictErrorChecking(final boolean strictErrorChecking) {
512         throw new UnsupportedOperationException("HtmlPage.setStrictErrorChecking is not yet implemented.");
513     }
514 
515     /**
516      * {@inheritDoc}
517      * Not yet implemented.
518      */
519     @Override
520     public String getDocumentURI() {
521         throw new UnsupportedOperationException("HtmlPage.getDocumentURI is not yet implemented.");
522     }
523 
524     /**
525      * {@inheritDoc}
526      * Not yet implemented.
527      */
528     @Override
529     public void setDocumentURI(final String documentURI) {
530         throw new UnsupportedOperationException("HtmlPage.setDocumentURI is not yet implemented.");
531     }
532 
533     /**
534      * {@inheritDoc}
535      * Not yet implemented.
536      */
537     @Override
538     public org.w3c.dom.Node adoptNode(final org.w3c.dom.Node source) throws DOMException {
539         throw new UnsupportedOperationException("HtmlPage.adoptNode is not yet implemented.");
540     }
541 
542     /**
543      * {@inheritDoc}
544      * Not yet implemented.
545      */
546     @Override
547     public DOMConfiguration getDomConfig() {
548         throw new UnsupportedOperationException("HtmlPage.getDomConfig is not yet implemented.");
549     }
550 
551     /**
552      * {@inheritDoc}
553      * Not yet implemented.
554      */
555     @Override
556     public org.w3c.dom.Node renameNode(final org.w3c.dom.Node newNode, final String namespaceURI,
557         final String qualifiedName) throws DOMException {
558         throw new UnsupportedOperationException("HtmlPage.renameNode is not yet implemented.");
559     }
560 
561     /**
562      * {@inheritDoc}
563      */
564     @Override
565     public Charset getCharset() {
566         if (originalCharset_ == null) {
567             originalCharset_ = getWebResponse().getContentCharset();
568         }
569         return originalCharset_;
570     }
571 
572     /**
573      * {@inheritDoc}
574      */
575     @Override
576     public String getContentType() {
577         return getWebResponse().getContentType();
578     }
579 
580     /**
581      * {@inheritDoc}
582      * Not yet implemented.
583      */
584     @Override
585     public DOMImplementation getImplementation() {
586         throw new UnsupportedOperationException("HtmlPage.getImplementation is not yet implemented.");
587     }
588 
589     /**
590      * {@inheritDoc}
591      * @param tagName the tag name, preferably in lowercase
592      */
593     @Override
594     public DomElement createElement(String tagName) {
595         if (tagName.indexOf(':') == -1) {
596             tagName = org.htmlunit.util.StringUtils.toRootLowerCase(tagName);
597         }
598         return getWebClient().getPageCreator().getHtmlParser().getFactory(tagName)
599                     .createElementNS(this, null, tagName, null);
600     }
601 
602     /**
603      * {@inheritDoc}
604      */
605     @Override
606     public DomElement createElementNS(final String namespaceURI, final String qualifiedName) {
607         return getWebClient().getPageCreator().getHtmlParser()
608                 .getElementFactory(this, namespaceURI, qualifiedName, false, true)
609                 .createElementNS(this, namespaceURI, qualifiedName, null);
610     }
611 
612     /**
613      * {@inheritDoc}
614      * <p>Unlike {@link #createAttribute(String)} the name is not lower cased.</p>
615      */
616     @Override
617     public DomAttr createAttributeNS(final String namespaceURI, final String qualifiedName) {
618         // an empty namespace is the same as no namespace
619         final String namespace = org.htmlunit.util.StringUtils.isEmptyOrNull(namespaceURI) ? null : namespaceURI;
620 
621         final int colonPosition = qualifiedName.indexOf(':');
622         final String prefix = colonPosition == -1 ? null : qualifiedName.substring(0, colonPosition);
623         final boolean xmlns = XMLConstants.XMLNS_ATTRIBUTE.equals(prefix)
624                 || XMLConstants.XMLNS_ATTRIBUTE.equals(qualifiedName);
625 
626         if ((prefix != null && namespace == null)
627                 || (XMLConstants.XML_NS_PREFIX.equals(prefix) && !XMLConstants.XML_NS_URI.equals(namespace))
628                 || (xmlns && !XMLConstants.XMLNS_ATTRIBUTE_NS_URI.equals(namespace))
629                 || (!xmlns && XMLConstants.XMLNS_ATTRIBUTE_NS_URI.equals(namespace))) {
630             throw new DOMException(DOMException.NAMESPACE_ERR,
631                     "createAttributeNS: invalid namespace '" + namespaceURI + "' for '" + qualifiedName + "'");
632         }
633 
634         return new DomAttr(this, namespace, qualifiedName, "", false);
635     }
636 
637     /**
638      * {@inheritDoc}
639      * <p>Entity references are not supported by HTML documents (DOM Level 2 Core). They are removed
640      * from the DOM standard and the browsers do not offer {@code createEntityReference}.
641      * </p>
642      */
643     @Override
644     public EntityReference createEntityReference(final String name) {
645         throw new DOMException(DOMException.NOT_SUPPORTED_ERR,
646                 "HtmlPage.createEntityReference: entity references are not supported by HTML documents.");
647     }
648 
649     /**
650      * {@inheritDoc}
651      * <p>DOM Level 2 Core forbids processing instructions in HTML documents, but the DOM standard and the
652      * browsers allow them and {@code document.createProcessingInstruction()} relies on this.
653      * </p>
654      */
655     @Override
656     public DomProcessingInstruction createProcessingInstruction(final String target, final String data) {
657         return new DomProcessingInstruction(this, target, data);
658     }
659 
660     /**
661      * {@inheritDoc}
662      */
663     @Override
664     public DomElement getElementById(final String elementId) {
665         if (elementId != null) {
666             ensureMappedElementsBuilt();
667             final MappedElementIndexEntry elements = idMap_.get(elementId);
668             if (elements != null) {
669                 return elements.first();
670             }
671         }
672         return null;
673     }
674 
675     /**
676      * Returns the {@link HtmlAnchor} with the specified name.
677      *
678      * @param name the name to search by
679      * @return the {@link HtmlAnchor} with the specified name
680      * @throws ElementNotFoundException if the anchor could not be found
681      */
682     public HtmlAnchor getAnchorByName(final String name) throws ElementNotFoundException {
683         return getDocumentElement().getOneHtmlElementByAttribute("a", DomElement.NAME_ATTRIBUTE, name);
684     }
685 
686     /**
687      * Returns the {@link HtmlAnchor} with the specified href.
688      *
689      * @param href the string to search by
690      * @return the HtmlAnchor
691      * @throws ElementNotFoundException if the anchor could not be found
692      */
693     public HtmlAnchor getAnchorByHref(final String href) throws ElementNotFoundException {
694         return getDocumentElement().getOneHtmlElementByAttribute("a", "href", href);
695     }
696 
697     /**
698      * Returns a list of all anchors contained in this page.
699      * @return the list of {@link HtmlAnchor} in this page
700      */
701     public List<HtmlAnchor> getAnchors() {
702         return getDocumentElement().getElementsByTagNameImpl("a");
703     }
704 
705     /**
706      * Returns the first anchor with the specified text.
707      * @param text the text to search for
708      * @return the first anchor that was found
709      * @throws ElementNotFoundException if no anchors are found with the specified text
710      */
711     public HtmlAnchor getAnchorByText(final String text) throws ElementNotFoundException {
712         WebAssert.notNull("text", text);
713 
714         for (final HtmlAnchor anchor : getAnchors()) {
715             if (text.equals(anchor.asNormalizedText())) {
716                 return anchor;
717             }
718         }
719         throw new ElementNotFoundException("a", "<text>", text);
720     }
721 
722     /**
723      * Returns the first form that matches the specified name.
724      * @param name the name to search for
725      * @return the first form
726      * @throws ElementNotFoundException If no forms match the specified result.
727      */
728     public HtmlForm getFormByName(final String name) throws ElementNotFoundException {
729         final List<HtmlForm> forms = getDocumentElement()
730                 .getElementsByAttribute("form", DomElement.NAME_ATTRIBUTE, name);
731         if (forms.isEmpty()) {
732             throw new ElementNotFoundException("form", DomElement.NAME_ATTRIBUTE, name);
733         }
734         return forms.get(0);
735     }
736 
737     /**
738      * Returns a list of all the forms in this page.
739      * @return all the forms in this page
740      */
741     public List<HtmlForm> getForms() {
742         return getDocumentElement().getElementsByTagNameImpl("form");
743     }
744 
745     /**
746      * Given a relative URL (ie <code>/foo</code>), returns a fully-qualified URL based on
747      * the URL that was used to load this page.
748      *
749      * @param relativeUrl the relative URL
750      * @return the fully-qualified URL for the specified relative URL
751      * @throws MalformedURLException if an error occurred when creating a URL object
752      */
753     public URL getFullyQualifiedUrl(String relativeUrl) throws MalformedURLException {
754         // to handle http: and http:/ in FF (Bug #474)
755         boolean incorrectnessNotified = false;
756         while (relativeUrl.startsWith("http:") && !relativeUrl.startsWith("http://")) {
757             if (!incorrectnessNotified) {
758                 notifyIncorrectness("Incorrect URL \"" + relativeUrl + "\" has been corrected");
759                 incorrectnessNotified = true;
760             }
761             relativeUrl = "http:/" + relativeUrl.substring(5);
762         }
763 
764         return WebClient.expandUrl(getBaseURL(), relativeUrl);
765     }
766 
767     /**
768      * Given a target attribute value, resolve the target using a base target for the page.
769      *
770      * @param elementTarget the target specified as an attribute of the element
771      * @return the resolved target to use for the element
772      */
773     public String getResolvedTarget(final String elementTarget) {
774         final String resolvedTarget;
775         if (base_ == null) {
776             resolvedTarget = elementTarget;
777         }
778         else if (elementTarget != null && !elementTarget.isEmpty()) {
779             resolvedTarget = elementTarget;
780         }
781         else {
782             resolvedTarget = base_.getTargetAttribute();
783         }
784         return resolvedTarget;
785     }
786 
787     /**
788      * Returns a list of ids (strings) that correspond to the tabbable elements
789      * in this page. Return them in the same order specified in {@link #getTabbableElements}
790      *
791      * @return the list of id's
792      */
793     public List<String> getTabbableElementIds() {
794         final List<String> list = new ArrayList<>();
795 
796         for (final HtmlElement element : getTabbableElements()) {
797             list.add(element.getId());
798         }
799 
800         return Collections.unmodifiableList(list);
801     }
802 
803     /**
804      * Returns a list of all elements that are tabbable in the order that will
805      * be used for tabbing.
806      * <p>
807      * The rules for determining tab order are as follows:
808      * </p>
809      * <ol>
810      *   <li>Those elements that support the tabindex attribute and assign a
811      *   positive value to it are navigated first. Navigation proceeds from the
812      *   element with the lowest tabindex value to the element with the highest
813      *   value. Values need not be sequential nor must they begin with any
814      *   particular value. Elements that have identical tabindex values should
815      *   be navigated in the order they appear in the character stream.</li>
816      *   <li>Those elements that do not support the tabindex attribute or
817      *   support it and assign it a value of "0" are navigated next. These
818      *   elements are navigated in the order they appear in the character
819      *   stream.</li>
820      *   <li>Elements that are disabled do not participate in the tabbing
821      *   order.</li>
822      * </ol>
823      * <p>
824      * Additionally, the value of tabindex must be within 0 and 32767. Any
825      * values outside this range will be ignored.
826      * </p>
827      * <p>
828      * The following elements support the <code>tabindex</code> attribute:
829      * A, AREA, BUTTON, INPUT, OBJECT, SELECT, and TEXTAREA.
830      * </p>
831      *
832      * @return all the tabbable elements in proper tab order
833      */
834     public List<HtmlElement> getTabbableElements() {
835         final List<HtmlElement> tabbableElements = new ArrayList<>();
836         for (final HtmlElement element : getHtmlElementDescendants()) {
837             final String tagName = element.getTagName();
838             if (TABBABLE_TAGS.contains(tagName)) {
839                 final boolean disabled = element.isDisabledElementAndDisabled();
840                 if (!disabled && !HtmlElement.TAB_INDEX_OUT_OF_BOUNDS.equals(element.getTabIndex())) {
841                     tabbableElements.add(element);
842                 }
843             }
844         }
845         tabbableElements.sort(createTabOrderComparator());
846         return Collections.unmodifiableList(tabbableElements);
847     }
848 
849     private static Comparator<HtmlElement> createTabOrderComparator() {
850         return (element1, element2) -> {
851             final Short i1 = element1.getTabIndex();
852             final Short i2 = element2.getTabIndex();
853 
854             final short index1;
855             if (i1 == null) {
856                 index1 = -1;
857             }
858             else {
859                 index1 = i1.shortValue();
860             }
861 
862             final short index2;
863             if (i2 == null) {
864                 index2 = -1;
865             }
866             else {
867                 index2 = i2.shortValue();
868             }
869 
870             final int result;
871             if (index1 > 0 && index2 > 0) {
872                 result = index1 - index2;
873             }
874             else if (index1 > 0) {
875                 result = -1;
876             }
877             else if (index2 > 0) {
878                 result = 1;
879             }
880             else if (index1 == index2) {
881                 result = 0;
882             }
883             else {
884                 result = index2 - index1;
885             }
886 
887             return result;
888         };
889     }
890 
891     /**
892      * Returns the HTML element that is assigned to the specified access key. An
893      * access key (aka mnemonic key) is used for keyboard navigation of the
894      * page.
895      * <p>
896      * Only the following HTML elements may have <code>accesskey</code>s defined: A, AREA,
897      * BUTTON, INPUT, LABEL, LEGEND, and TEXTAREA.
898      * </p>
899      *
900      * @param accessKey the key to look for
901      * @return the HTML element that is assigned to the specified key or null
902      *      if no elements can be found that match the specified key.
903      */
904     public HtmlElement getHtmlElementByAccessKey(final char accessKey) {
905         final List<HtmlElement> elements = getHtmlElementsByAccessKey(accessKey);
906         if (elements.isEmpty()) {
907             return null;
908         }
909         return elements.get(0);
910     }
911 
912     /**
913      * Returns all the HTML elements that are assigned to the specified access key. An
914      * access key (aka mnemonic key) is used for keyboard navigation of the
915      * page.
916      * <p>
917      * The HTML specification seems to indicate that one accesskey cannot be used
918      * for multiple elements however Internet Explorer does seem to support this.
919      * It's worth noting that Firefox does not support multiple elements with one
920      * access key so you are making your HTML browser specific if you rely on this
921      * feature.
922      * </p>
923      *
924      * <p>
925      * Only the following HTML elements may have <code>accesskey</code>s defined: A, AREA,
926      * BUTTON, INPUT, LABEL, LEGEND, and TEXTAREA.
927      * </p>
928      *
929      * @param accessKey the key to look for
930      * @return the elements that are assigned to the specified accesskey
931      */
932     public List<HtmlElement> getHtmlElementsByAccessKey(final char accessKey) {
933         final List<HtmlElement> elements = new ArrayList<>();
934 
935         final String searchString = Character.toString(accessKey).toLowerCase(Locale.ROOT);
936         for (final HtmlElement element : getHtmlElementDescendants()) {
937             if (ACCEPTABLE_TAG_NAMES.contains(element.getTagName())) {
938                 final String accessKeyAttribute = element.getAttributeDirect("accesskey");
939                 if (searchString.equalsIgnoreCase(accessKeyAttribute)) {
940                     elements.add(element);
941                 }
942             }
943         }
944 
945         return elements;
946     }
947 
948     /**
949      * <p>Executes the specified JavaScript code within the page. The usage would be similar to what can
950      * be achieved to execute JavaScript in the current page by entering "javascript:...some JS code..."
951      * in the URL field of a native browser.</p>
952      * <p><b>Note:</b> the provided code won't be executed if JavaScript has been disabled on the WebClient
953      * (see {@link org.htmlunit.WebClient#isJavaScriptEnabled()}).</p>
954      * @param sourceCode the JavaScript code to execute
955      * @return a ScriptResult which will contain both the current page (which may be different from
956      *         the previous page) and a JavaScript result object
957      */
958     public ScriptResult executeJavaScript(final String sourceCode) {
959         return executeJavaScript(sourceCode, "injected script", 1);
960     }
961 
962     /**
963      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
964      * <p>
965      * Execute the specified JavaScript if a JavaScript engine was successfully
966      * instantiated. If this JavaScript causes the current page to be reloaded
967      * (through location="" or form.submit()) then return the new page, otherwise
968      * return the current page.
969      * </p>
970      * <p><b>Please note:</b> Although this method is public, it is not intended for
971      * general execution of JavaScript. Users of HtmlUnit should interact with the pages
972      * as a user would by clicking on buttons or links and having the JavaScript event
973      * handlers execute as needed.
974      * </p>
975      *
976      * @param sourceCode the JavaScript code to execute
977      * @param sourceName the name for this chunk of code (will be displayed in error messages)
978      * @param startLine the line at which the script source starts
979      * @return a ScriptResult which will contain both the current page (which may be different from
980      *         the previous page) and a JavaScript result object.
981      */
982     public ScriptResult executeJavaScript(String sourceCode, final String sourceName, final int startLine) {
983         if (!getWebClient().isJavaScriptEnabled()) {
984             return new ScriptResult(JavaScriptEngine.UNDEFINED);
985         }
986 
987         if (org.htmlunit.util.StringUtils.startsWithIgnoreCase(sourceCode,
988                                                 JavaScriptURLConnection.JAVASCRIPT_PREFIX)) {
989             sourceCode = sourceCode.substring(JavaScriptURLConnection.JAVASCRIPT_PREFIX.length()).trim();
990             if (sourceCode.startsWith("return ")) {
991                 sourceCode = sourceCode.substring("return ".length());
992             }
993         }
994 
995         final Window window = getEnclosingWindow().getScriptableObject();
996         final VarScope scope = ScriptableObject.getTopLevelScope(window.getParentScope());
997 
998         final Object result = getWebClient().getJavaScriptEngine()
999                 .execute(this, scope, sourceCode, sourceName, startLine);
1000         return new ScriptResult(result);
1001     }
1002 
1003     /** Various possible external JavaScript file loading results. */
1004     enum JavaScriptLoadResult {
1005         /** The load was aborted and nothing was done. */
1006         NOOP,
1007         /** The load was aborted and nothing was done. */
1008         NO_CONTENT,
1009         /** The external JavaScript file was downloaded and compiled successfully. */
1010         SUCCESS,
1011         /** The external JavaScript file was not downloaded successfully. */
1012         DOWNLOAD_ERROR,
1013         /** The external JavaScript file was downloaded but was not compiled successfully. */
1014         COMPILATION_ERROR
1015     }
1016 
1017     /**
1018      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1019      *
1020      * @param srcAttribute the source attribute from the script tag
1021      * @param scriptCharset the charset from the script tag
1022      * @return the result of loading the specified external JavaScript file
1023      * @throws FailingHttpStatusCodeException if the request's status code indicates a request
1024      *         failure and the {@link WebClient} was configured to throw exceptions on failing
1025      *         HTTP status codes
1026      */
1027     JavaScriptLoadResult loadExternalJavaScriptFile(final String srcAttribute,
1028                             final Charset scriptCharset, final boolean crossorigin)
1029         throws FailingHttpStatusCodeException {
1030 
1031         final WebClient client = getWebClient();
1032         if (org.htmlunit.util.StringUtils.isBlank(srcAttribute) || !client.isJavaScriptEnabled()) {
1033             return JavaScriptLoadResult.NOOP;
1034         }
1035 
1036         final URL scriptURL;
1037         try {
1038             scriptURL = getFullyQualifiedUrl(srcAttribute);
1039             final String protocol = scriptURL.getProtocol();
1040             if ("javascript".equals(protocol)) {
1041                 if (LOG.isInfoEnabled()) {
1042                     LOG.info("Ignoring script src [" + srcAttribute + "]");
1043                 }
1044                 return JavaScriptLoadResult.NOOP;
1045             }
1046             if (!"http".equals(protocol) && !"https".equals(protocol)
1047                     && !"data".equals(protocol) && !"file".equals(protocol)) {
1048                 client.getJavaScriptErrorListener().malformedScriptURL(this, srcAttribute,
1049                         new MalformedURLException("unknown protocol: '" + protocol + "'"));
1050                 return JavaScriptLoadResult.NOOP;
1051             }
1052         }
1053         catch (final MalformedURLException e) {
1054             client.getJavaScriptErrorListener().malformedScriptURL(this, srcAttribute, e);
1055             return JavaScriptLoadResult.NOOP;
1056         }
1057 
1058         final Object script;
1059         try {
1060             script = loadJavaScriptFromUrl(scriptURL, scriptCharset, crossorigin);
1061         }
1062         catch (final IOException e) {
1063             client.getJavaScriptErrorListener().loadScriptError(this, scriptURL, e);
1064             return JavaScriptLoadResult.DOWNLOAD_ERROR;
1065         }
1066         catch (final FailingHttpStatusCodeException e) {
1067             if (e.getStatusCode() == HttpStatus.NO_CONTENT_204) {
1068                 return JavaScriptLoadResult.NO_CONTENT;
1069             }
1070             client.getJavaScriptErrorListener().loadScriptError(this, scriptURL, e);
1071             throw e;
1072         }
1073 
1074         if (script == null) {
1075             return JavaScriptLoadResult.COMPILATION_ERROR;
1076         }
1077 
1078         final Window window = getEnclosingWindow().getScriptableObject();
1079         final VarScope scope = ScriptableObject.getTopLevelScope(window.getParentScope());
1080 
1081         @SuppressWarnings("unchecked")
1082         final AbstractJavaScriptEngine<Object> engine = (AbstractJavaScriptEngine<Object>) client.getJavaScriptEngine();
1083         engine.execute(this, scope, script);
1084         return JavaScriptLoadResult.SUCCESS;
1085     }
1086 
1087     /**
1088      * Loads JavaScript from the specified URL. This method may return {@code null} if
1089      * there is a problem loading the code from the specified URL.
1090      *
1091      * @param url the URL of the script
1092      * @param scriptCharset the charset from the script tag
1093      * @return the content of the file, or {@code null} if we ran into a compile error
1094      * @throws IOException if there is a problem downloading the JavaScript file
1095      * @throws FailingHttpStatusCodeException if the request's status code indicates a request
1096      *         failure and the {@link WebClient} was configured to throw exceptions on failing
1097      *         HTTP status codes
1098      */
1099     private Object loadJavaScriptFromUrl(final URL url, final Charset scriptCharset,
1100                     final boolean crossorigin) throws IOException,
1101         FailingHttpStatusCodeException {
1102 
1103         final WebRequest referringRequest = getWebResponse().getWebRequest();
1104 
1105         final WebClient client = getWebClient();
1106         final WebRequest request = new WebRequest(url);
1107         // copy all headers from the referring request
1108         request.setAdditionalHeaders(new HashMap<>(referringRequest.getAdditionalHeaders()));
1109         if (!UrlUtils.isSameOrigin(referringRequest.getUrl(), url)) {
1110             // credentials are not forwarded to another origin
1111             request.removeAdditionalHeader("Authorization");
1112         }
1113 
1114         // at least overwrite this headers
1115         request.setAdditionalHeader(HttpHeader.ACCEPT, client.getBrowserVersion().getScriptAcceptHeader());
1116 
1117         request.setFetchDestination(WebRequest.FetchDestination.SCRIPT);
1118         request.setRequestingUrl(referringRequest.getUrl());
1119         request.setFetchModeOverride(WebRequest.FetchMode.NO_CORS);
1120 
1121         request.setRefererHeader(referringRequest.getUrl());
1122         request.setCharset(scriptCharset);
1123 
1124         // use info from script tag or fall back to utf-8
1125         // https://www.rfc-editor.org/rfc/rfc9239#section-4.2
1126         if (scriptCharset != null) {
1127             request.setDefaultResponseContentCharset(scriptCharset);
1128         }
1129         else {
1130             request.setDefaultResponseContentCharset(StandardCharsets.UTF_8);
1131         }
1132 
1133         if (crossorigin) {
1134             request.setFetchModeOverride(WebRequest.FetchMode.CORS);
1135 
1136             if (client.getBrowserVersion().hasFeature(HTTP_HEADER_CH_UA)) {
1137                 request.setAdditionalHeader(HttpHeader.ORIGIN,
1138                         UrlUtils.getUrlWithProtocolAndAuthority(url).toExternalForm());
1139             }
1140         }
1141 
1142         // our cache is a bit strange;
1143         // loadWebResponse check the cache for the web response
1144         // AND also fixes the request url for the following cache lookups
1145         final WebResponse response = client.loadWebResponse(request);
1146 
1147         // now we can look into the cache with the fixed request for
1148         // a cached script
1149         final Cache cache = client.getCache();
1150         final Object cachedScript = cache.getCachedObject(request);
1151         if (cachedScript instanceof Script) {
1152             return cachedScript;
1153         }
1154 
1155         client.printContentIfNecessary(response);
1156         client.throwFailingHttpStatusCodeExceptionIfNecessary(response);
1157 
1158         final int statusCode = response.getStatusCode();
1159         if (statusCode == HttpStatus.NO_CONTENT_204) {
1160             throw new FailingHttpStatusCodeException(response);
1161         }
1162 
1163         if (!response.isSuccess()) {
1164             throw new IOException("Unable to download JavaScript from '" + url + "' (status " + statusCode + ").");
1165         }
1166 
1167         final String contentType = response.getContentType();
1168         if (contentType != null) {
1169             if (MimeType.isObsoleteJavascriptMimeType(contentType)) {
1170                 getWebClient().getIncorrectnessListener().notify(
1171                         "Obsolete content type encountered: '" + contentType + "' "
1172                                 + "for remotely loaded JavaScript element at '" + url + "'.", this);
1173             }
1174             else if (!MimeType.isJavascriptMimeType(contentType)) {
1175                 getWebClient().getIncorrectnessListener().notify(
1176                         "Expect content type of '" + MimeType.TEXT_JAVASCRIPT + "' "
1177                                 + "for remotely loaded JavaScript element at '" + url + "', "
1178                                 + "but got '" + contentType + "'.", this);
1179             }
1180         }
1181 
1182         final Charset scriptEncoding = response.getContentCharset();
1183         final String scriptCode = response.getContentAsString(scriptEncoding);
1184         if (null != scriptCode) {
1185             final AbstractJavaScriptEngine<?> javaScriptEngine = client.getJavaScriptEngine();
1186 
1187             final Window window = getEnclosingWindow().getScriptableObject();
1188             final VarScope scope = ScriptableObject.getTopLevelScope(window.getParentScope());
1189 
1190             final Object script = javaScriptEngine.compile(this, scope, scriptCode, url.toExternalForm(), 1);
1191             if (script != null && cache.cacheIfPossible(request, response, script)) {
1192                 // no cleanup if the response is stored inside the cache
1193                 return script;
1194             }
1195 
1196             response.cleanUp();
1197             return script;
1198         }
1199 
1200         response.cleanUp();
1201         return null;
1202     }
1203 
1204     /**
1205      * Returns the title like {@code document.title} does: the text of the first title element
1206      * with ASCII whitespace stripped and collapsed. Other spaces, including the non-breaking space, are kept.
1207      *
1208      * @return the title like {@code document.title}
1209      */
1210     public String getDocumentTitle() {
1211         final HtmlTitle title = getTitleElement();
1212         if (title == null) {
1213             return "";
1214         }
1215 
1216         final String titleTextContent = title.getText();
1217         final StringBuilder sb = new StringBuilder(titleTextContent.length());
1218         boolean pendingSpace = false;
1219         for (int i = 0; i < titleTextContent.length(); i++) {
1220             final char c = titleTextContent.charAt(i);
1221             if (c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\r') {
1222                 pendingSpace = sb.length() > 0;
1223             }
1224             else {
1225                 if (pendingSpace) {
1226                     sb.append(' ');
1227                     pendingSpace = false;
1228                 }
1229                 sb.append(c);
1230             }
1231         }
1232         return sb.toString();
1233     }
1234 
1235     /**
1236      * Returns the normalized title of this page, or an empty string if no title element is present.
1237      * <p>
1238      * The returned text is normalized by collapsing consecutive whitespace characters
1239      * (including newlines and tabs) into a single space and trimming leading and trailing whitespace.
1240      * </p>
1241      *
1242      * @return the normalized page title, or an empty string if the title was not specified
1243      */
1244     public String getTitleText() {
1245         final HtmlTitle titleElement = getTitleElement();
1246         if (titleElement != null) {
1247             return titleElement.asNormalizedText();
1248         }
1249         return "";
1250     }
1251 
1252     /**
1253      * Sets the text for the title of this page. If there is not a title element
1254      * on this page, then one has to be generated.
1255      * @param message the new text
1256      */
1257     public void setTitleText(final String message) {
1258         HtmlTitle titleElement = getTitleElement();
1259         if (titleElement == null) {
1260             LOG.debug("No title element, creating one");
1261             final HtmlHead head = (HtmlHead) getFirstChildElement(getDocumentElement(), HtmlHead.class);
1262             if (head == null) {
1263                 // perhaps should we create head too?
1264                 throw new IllegalStateException("Headelement was not defined for this page");
1265             }
1266             final Map<String, DomAttr> emptyMap = Collections.emptyMap();
1267             titleElement = new HtmlTitle(HtmlTitle.TAG_NAME, this, emptyMap);
1268             if (head.getFirstChild() != null) {
1269                 head.getFirstChild().insertBefore(titleElement);
1270             }
1271             else {
1272                 head.appendChild(titleElement);
1273             }
1274         }
1275 
1276         titleElement.setText(message);
1277     }
1278 
1279     /**
1280      * Gets the first child of startElement that is an instance of the given class.
1281      * @param startElement the parent element
1282      * @param clazz the class to search for
1283      * @return {@code null} if no child found
1284      */
1285     private static DomElement getFirstChildElement(final DomElement startElement, final Class<?> clazz) {
1286         if (startElement == null) {
1287             return null;
1288         }
1289         for (final DomElement element : startElement.getChildElements()) {
1290             if (clazz.isInstance(element)) {
1291                 return element;
1292             }
1293         }
1294 
1295         return null;
1296     }
1297 
1298     /**
1299      * Gets the first child of startElement or it's children that is an instance of the given class.
1300      * @param startElement the parent element
1301      * @param clazz the class to search for
1302      * @return {@code null} if no child found
1303      */
1304     private DomElement getFirstChildElementRecursive(final DomElement startElement, final Class<?> clazz) {
1305         if (startElement == null) {
1306             return null;
1307         }
1308         for (final DomElement element : startElement.getChildElements()) {
1309             if (clazz.isInstance(element)) {
1310                 return element;
1311             }
1312             final DomElement childFound = getFirstChildElementRecursive(element, clazz);
1313             if (childFound != null) {
1314                 return childFound;
1315             }
1316         }
1317 
1318         return null;
1319     }
1320 
1321     /**
1322      * Gets the title element for this page. Returns null if one is not found.
1323      *
1324      * @return the title element for this page or null if this is not one
1325      */
1326     private HtmlTitle getTitleElement() {
1327         return (HtmlTitle) getFirstChildElementRecursive(getDocumentElement(), HtmlTitle.class);
1328     }
1329 
1330     /**
1331      * Looks for and executes any appropriate event handlers. Looks for body and frame tags.
1332      * @param eventType either {@link Event#TYPE_LOAD}, {@link Event#TYPE_UNLOAD}, or {@link Event#TYPE_BEFORE_UNLOAD}
1333      * @return {@code true} if user accepted <code>onbeforeunload</code> (not relevant to other events)
1334      */
1335     private boolean executeEventHandlersIfNeeded(final String eventType) {
1336         // If JavaScript isn't enabled, there's nothing for us to do.
1337         if (!getWebClient().isJavaScriptEnabled()) {
1338             return true;
1339         }
1340 
1341         // Execute the specified event on the document element.
1342         final WebWindow window = getEnclosingWindow();
1343         if (window.getScriptableObject() instanceof Window) {
1344             final Event event;
1345             if (Event.TYPE_BEFORE_UNLOAD.equals(eventType)) {
1346                 event = new BeforeUnloadEvent(this, eventType);
1347             }
1348             else {
1349                 event = new Event(this, eventType);
1350             }
1351 
1352             // This is the same as DomElement.fireEvent() and was copied
1353             // here so it could be used with HtmlPage.
1354             if (LOG.isDebugEnabled()) {
1355                 LOG.debug("Firing " + event);
1356             }
1357 
1358             final EventTarget jsNode;
1359             if (Event.TYPE_DOM_DOCUMENT_LOADED.equals(eventType)) {
1360                 jsNode = getScriptableObject();
1361             }
1362             else if (Event.TYPE_READY_STATE_CHANGE.equals(eventType)) {
1363                 jsNode = getDocumentElement().getScriptableObject();
1364             }
1365             else {
1366                 // The load/beforeunload/unload events target Document but paths Window only (tested in Chrome/FF)
1367                 jsNode = window.getScriptableObject();
1368             }
1369 
1370             ((JavaScriptEngine) getWebClient().getJavaScriptEngine()).callSecured(cx -> jsNode.fireEvent(event), this);
1371 
1372             if (!isOnbeforeunloadAccepted(this, event)) {
1373                 return false;
1374             }
1375         }
1376 
1377         // If this page was loaded in a frame, execute the version of the event specified on the frame tag.
1378         if (window instanceof FrameWindow fw) {
1379             final BaseFrameElement frame = fw.getFrameElement();
1380 
1381             // if part of a document fragment, then the load event is not triggered
1382             if (Event.TYPE_LOAD.equals(eventType) && frame.getParentNode() instanceof DomDocumentFragment) {
1383                 return true;
1384             }
1385 
1386             if (frame.hasEventHandlers("on" + eventType)) {
1387                 if (LOG.isDebugEnabled()) {
1388                     LOG.debug("Executing on" + eventType + " handler for " + frame);
1389                 }
1390                 if (window.getScriptableObject() instanceof Window) {
1391                     final Event event;
1392                     if (Event.TYPE_BEFORE_UNLOAD.equals(eventType)) {
1393                         event = new BeforeUnloadEvent(frame, eventType);
1394                     }
1395                     else {
1396                         event = new Event(frame, eventType);
1397                     }
1398                     // This fires the "load" event for the <frame> element which, like all non-window
1399                     // load events, propagates up to Document but not Window.  The "load" event for
1400                     // <frameset> on the other hand, like that of <body>, is handled above where it is
1401                     // fired against Document and directed to Window.
1402                     frame.fireEvent(event);
1403 
1404                     if (!isOnbeforeunloadAccepted((HtmlPage) frame.getPage(), event)) {
1405                         return false;
1406                     }
1407                 }
1408             }
1409         }
1410 
1411         return true;
1412     }
1413 
1414     /**
1415      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1416      *
1417      * @return true if the OnbeforeunloadHandler has accepted to change the page
1418      */
1419     public boolean isOnbeforeunloadAccepted() {
1420         return executeEventHandlersIfNeeded(Event.TYPE_BEFORE_UNLOAD);
1421     }
1422 
1423     private boolean isOnbeforeunloadAccepted(final HtmlPage page, final Event event) {
1424         if (event instanceof BeforeUnloadEvent beforeUnloadEvent) {
1425             if (beforeUnloadEvent.isBeforeUnloadMessageSet()) {
1426                 final OnbeforeunloadHandler handler = getWebClient().getOnbeforeunloadHandler();
1427                 if (handler == null) {
1428                     LOG.warn("document.onbeforeunload() returned a string in event.returnValue,"
1429                             + " but no onbeforeunload handler installed.");
1430                 }
1431                 else {
1432                     final String message = JavaScriptEngine.toString(beforeUnloadEvent.getReturnValue());
1433                     return handler.handleEvent(page, message);
1434                 }
1435             }
1436         }
1437         return true;
1438     }
1439 
1440     /**
1441      * If a refresh has been specified either through a meta tag or an HTTP
1442      * response header, then perform that refresh.
1443      * @throws IOException if an IO problem occurs
1444      */
1445     private void executeRefreshIfNeeded() throws IOException {
1446         // If this page is not in a frame then a refresh has already happened,
1447         // most likely through the JavaScript onload handler, so we don't do a
1448         // second refresh.
1449         final WebWindow window = getEnclosingWindow();
1450         if (window == null) {
1451             return;
1452         }
1453 
1454         final String refreshString = getRefreshStringOrNull();
1455         if (refreshString == null || refreshString.isEmpty()) {
1456             return;
1457         }
1458 
1459         final double time;
1460         final URL url;
1461 
1462         final int index = StringUtils.indexOfAnyBut(refreshString, "0123456789.");
1463 
1464         if (index == -1) {
1465             // Format: <meta http-equiv='refresh' content='10'>
1466             try {
1467                 time = Double.parseDouble(refreshString);
1468             }
1469             catch (final NumberFormatException e) {
1470                 if (LOG.isErrorEnabled()) {
1471                     LOG.error("Malformed refresh string (no ';' but not a number): " + refreshString, e);
1472                 }
1473                 return;
1474             }
1475             url = getUrl();
1476         }
1477         else {
1478             // Format: <meta http-equiv='refresh' content='10;url=http://www.blah.com'>
1479             try {
1480                 time = Double.parseDouble(refreshString.substring(0, index));
1481             }
1482             catch (final NumberFormatException e) {
1483                 if (LOG.isErrorEnabled()) {
1484                     LOG.error("Malformed refresh string (no valid number before ';') " + refreshString, e);
1485                 }
1486                 return;
1487             }
1488 
1489             String urlPart = refreshString.substring(index);
1490             final char separator = urlPart.charAt(0);
1491             if (";, \r\n\t".indexOf(separator) >= 0) {
1492                 urlPart = StringUtils.stripStart(urlPart, ";, \r\n\t");
1493                 urlPart = urlPart.toLowerCase(Locale.ROOT);
1494                 if (urlPart.startsWith("url")) {
1495                     urlPart = urlPart.substring(3);
1496                     urlPart = urlPart.trim();
1497 
1498                     if (urlPart.length() > 0 && urlPart.charAt(0) == '=') {
1499                         urlPart = urlPart.substring(1);
1500                         urlPart = urlPart.trim();
1501                     }
1502                 }
1503 
1504                 if (org.htmlunit.util.StringUtils.isBlank(urlPart)) {
1505                     //content='10; URL=' is treated as content='10'
1506                     url = getUrl();
1507                 }
1508                 else {
1509                     if (urlPart.charAt(0) == '"' || urlPart.charAt(0) == 0x27) {
1510                         urlPart = urlPart.substring(1);
1511                     }
1512                     if (urlPart.charAt(urlPart.length() - 1) == '"' || urlPart.charAt(urlPart.length() - 1) == 0x27) {
1513                         urlPart = urlPart.substring(0, urlPart.length() - 1);
1514                     }
1515                     try {
1516                         url = getFullyQualifiedUrl(urlPart);
1517                     }
1518                     catch (final MalformedURLException e) {
1519                         if (LOG.isErrorEnabled()) {
1520                             LOG.error("Malformed URL in refresh string: " + refreshString, e);
1521                         }
1522                         return;
1523                     }
1524                 }
1525             }
1526             else {
1527                 if (LOG.isErrorEnabled()) {
1528                     LOG.error("Malformed refresh string (separator after time missing): " + refreshString);
1529                 }
1530                 return;
1531             }
1532         }
1533 
1534         processRefresh(url, time);
1535     }
1536 
1537     // this is different from what is done in org.htmlunit.WebClient.loadWebResponseFromWebConnection(WebRequest, int)
1538     // because there we are directly replacing the response before loading the response into the window
1539     // here we are replacing the page in the window (maybe after some time)
1540     private void processRefresh(final URL url, final double time) throws IOException {
1541         final WebClient webClient = getWebClient();
1542 
1543         final int refreshLimit = webClient.getOptions().getPageRefreshLimit();
1544         if (refreshLimit == 0) {
1545             final WebResponse webResponse = getWebResponse();
1546             throw new FailingHttpStatusCodeException("Too many redirects for "
1547                     + webResponse.getWebRequest().getUrl(), webResponse);
1548         }
1549 
1550         if (refreshLimit >= 0) {
1551             final StackTraceElement[] elements = new Exception().getStackTrace();
1552             int count = 0;
1553             final int elementCountLimit = refreshLimit > 50 ? 400 : refreshLimit > 10 ? 80 : 5;
1554             final int elementCount = elements.length;
1555 
1556             if (elementCount > elementCountLimit) {
1557                 for (int i = 0; i < elementCount; i++) {
1558                     if ("processRefresh".equals(elements[i].getMethodName())
1559                             && "org.htmlunit.html.HtmlPage".equals(elements[i].getClassName())) {
1560                         count++;
1561                         if (count >= refreshLimit) {
1562                             final WebResponse webResponse = getWebResponse();
1563                             throw new FailingHttpStatusCodeException(
1564                                             "Too many redirects (>= " + count + ") for "
1565                                                 + webResponse.getWebRequest().getUrl(), webResponse);
1566                         }
1567                     }
1568                 }
1569             }
1570         }
1571 
1572         webClient.getRefreshHandler().handleRefresh(this, url, (int) time);
1573     }
1574 
1575     /**
1576      * Returns an auto-refresh string if specified. This will look in both the meta
1577      * tags and inside the HTTP response headers.
1578      * @return the auto-refresh string
1579      */
1580     private String getRefreshStringOrNull() {
1581         final List<HtmlMeta> metaTags = getMetaTags("refresh");
1582         if (!metaTags.isEmpty()) {
1583             return metaTags.get(0).getContentAttribute().trim();
1584         }
1585         return getWebResponse().getResponseHeaderValue("Refresh");
1586     }
1587 
1588     private void processPostponedActionsIfNeeded() {
1589         if (!getWebClient().isJavaScriptEnabled()) {
1590             return;
1591         }
1592         getWebClient().getJavaScriptEngine().processPostponedActions();
1593     }
1594 
1595     /**
1596      * Executes any deferred scripts, if necessary.
1597      */
1598     private void executeDeferredScriptsIfNeeded() {
1599         if (!getWebClient().isJavaScriptEnabled()) {
1600             return;
1601         }
1602         final DomElement doc = getDocumentElement();
1603         final List<HtmlScript> scripts = new ArrayList<>();
1604 
1605         // don't call getElementsByTagName() here because it creates a live collection
1606         for (final HtmlElement elem : doc.getHtmlElementDescendants()) {
1607             if ("script".equals(elem.getLocalName()) && (elem instanceof HtmlScript script)) {
1608                 if (script.isDeferred() && ATTRIBUTE_NOT_DEFINED != script.getSrcAttribute()) {
1609                     scripts.add(script);
1610                 }
1611             }
1612         }
1613         for (final HtmlScript script : scripts) {
1614             ScriptElementSupport.executeScriptIfNeeded(script, true, true);
1615         }
1616     }
1617 
1618     /**
1619      * Deregister frames that are no longer in use.
1620      */
1621     public void deregisterFramesIfNeeded() {
1622         final List<BaseFrameElement> frameElementsCopy = new ArrayList<>(frameElements_);
1623         for (final BaseFrameElement frameElement : frameElementsCopy) {
1624             final WebWindow window = frameElement.getEnclosedWindow();
1625             getWebClient().deregisterWebWindow(window);
1626             final Page page = window.getEnclosedPage();
1627             if (page != null && page.isHtmlPage()) {
1628                 // seems quite silly, but for instance if the src attribute of an iframe is not
1629                 // set, the error only occurs when leaving the page
1630                 ((HtmlPage) page).deregisterFramesIfNeeded();
1631             }
1632         }
1633     }
1634 
1635     /**
1636      * Returns a list containing all the frames (from frame and iframe tags) in this page
1637      * in document order.
1638      * @return a list of {@link FrameWindow}
1639      */
1640     public List<FrameWindow> getFrames() {
1641         final List<BaseFrameElement> frameElements = new ArrayList<>(frameElements_);
1642         frameElements.sort(DOCUMENT_POSITION_COMPERATOR);
1643 
1644         final List<FrameWindow> list = new ArrayList<>(frameElements.size());
1645         for (final BaseFrameElement frameElement : frameElements) {
1646             list.add(frameElement.getEnclosedWindow());
1647         }
1648         return list;
1649     }
1650 
1651     /**
1652      * Returns the first frame contained in this page with the specified name.
1653      * @param name the name to search for
1654      * @return the first frame found
1655      * @throws ElementNotFoundException If no frame exist in this page with the specified name.
1656      */
1657     public FrameWindow getFrameByName(final String name) throws ElementNotFoundException {
1658         for (final BaseFrameElement frameElement : frameElements_) {
1659             final FrameWindow fw = frameElement.getEnclosedWindow();
1660             if (fw.getName().equals(name)) {
1661                 return fw;
1662             }
1663         }
1664 
1665         throw new ElementNotFoundException("frame or iframe", DomElement.NAME_ATTRIBUTE, name);
1666     }
1667 
1668     /**
1669      * Simulate pressing an access key. This may change the focus, may click buttons and may invoke
1670      * JavaScript.
1671      *
1672      * @param accessKey the key that will be pressed
1673      * @return the element that has the focus after pressing this access key or null if no element
1674      *         has the focus.
1675      * @throws IOException if an IO error occurs during the processing of this access key (this
1676      *         would only happen if the access key triggered a button which in turn caused a page load)
1677      */
1678     public DomElement pressAccessKey(final char accessKey) throws IOException {
1679         final HtmlElement element = getHtmlElementByAccessKey(accessKey);
1680         if (element != null) {
1681             element.focus();
1682             if (element instanceof HtmlAnchor
1683                     || element instanceof HtmlArea
1684                     || element instanceof HtmlButton
1685                     || element instanceof HtmlInput
1686                     || element instanceof HtmlLabel
1687                     || element instanceof HtmlLegend
1688                     || element instanceof HtmlTextArea) {
1689                 final Page newPage = element.click();
1690 
1691                 if (newPage != this && getFocusedElement() == element) {
1692                     // The page was reloaded therefore no element on this page will have the focus.
1693                     getFocusedElement().blur();
1694                 }
1695             }
1696         }
1697 
1698         return getFocusedElement();
1699     }
1700 
1701     /**
1702      * Move the focus to the next element in the tab order. To determine the specified tab
1703      * order, refer to {@link HtmlPage#getTabbableElements()}
1704      *
1705      * @return the element that has focus after calling this method
1706      */
1707     public HtmlElement tabToNextElement() {
1708         final List<HtmlElement> elements = getTabbableElements();
1709         if (elements.isEmpty()) {
1710             setFocusedElement(null);
1711             return null;
1712         }
1713 
1714         final HtmlElement elementToGiveFocus;
1715         final DomElement elementWithFocus = getFocusedElement();
1716         if (elementWithFocus == null) {
1717             elementToGiveFocus = elements.get(0);
1718         }
1719         else {
1720             final int index = elements.indexOf(elementWithFocus);
1721             if (index == -1) {
1722                 // The element with focus isn't on this page
1723                 elementToGiveFocus = elements.get(0);
1724             }
1725             else if (index == elements.size() - 1) {
1726                 // if at last jump to start
1727                 elementToGiveFocus = elements.get(0);
1728             }
1729             else {
1730                 elementToGiveFocus = elements.get(index + 1);
1731             }
1732         }
1733 
1734         setFocusedElement(elementToGiveFocus);
1735         return elementToGiveFocus;
1736     }
1737 
1738     /**
1739      * Move the focus to the previous element in the tab order. To determine the specified tab
1740      * order, refer to {@link HtmlPage#getTabbableElements()}
1741      *
1742      * @return the element that has focus after calling this method
1743      */
1744     public HtmlElement tabToPreviousElement() {
1745         final List<HtmlElement> elements = getTabbableElements();
1746         if (elements.isEmpty()) {
1747             setFocusedElement(null);
1748             return null;
1749         }
1750 
1751         final HtmlElement elementToGiveFocus;
1752         final DomElement elementWithFocus = getFocusedElement();
1753         if (elementWithFocus == null) {
1754             elementToGiveFocus = elements.get(elements.size() - 1);
1755         }
1756         else {
1757             final int index = elements.indexOf(elementWithFocus);
1758             if (index == -1) {
1759                 // The element with focus isn't on this page
1760                 elementToGiveFocus = elements.get(elements.size() - 1);
1761             }
1762             else if (index == 0) {
1763                 // first; back to the last
1764                 elementToGiveFocus = elements.get(elements.size() - 1);
1765             }
1766             else {
1767                 elementToGiveFocus = elements.get(index - 1);
1768             }
1769         }
1770 
1771         setFocusedElement(elementToGiveFocus);
1772         return elementToGiveFocus;
1773     }
1774 
1775     /**
1776      * Returns the HTML element with the specified ID. If more than one element
1777      * has this ID (not allowed by the HTML spec), then this method returns the
1778      * first one.
1779      *
1780      * @param elementId the ID value to search for
1781      * @param <E> the element type
1782      * @return the HTML element with the specified ID
1783      * @throws ElementNotFoundException if no element was found matching the specified ID
1784      */
1785     @SuppressWarnings("unchecked")
1786     public <E extends HtmlElement> E getHtmlElementById(final String elementId) throws ElementNotFoundException {
1787         final DomElement element = getElementById(elementId);
1788         if (element == null) {
1789             throw new ElementNotFoundException("*", DomElement.ID_ATTRIBUTE, elementId);
1790         }
1791         return (E) element;
1792     }
1793 
1794     /**
1795      * Returns the elements with the specified ID. If there are no elements
1796      * with the specified ID, this method returns an empty list. Please note that
1797      * the lists returned by this method are immutable.
1798      *
1799      * @param elementId the ID value to search for
1800      * @return the elements with the specified name attribute
1801      */
1802     public List<DomElement> getElementsById(final String elementId) {
1803         if (elementId != null) {
1804             ensureMappedElementsBuilt();
1805             final MappedElementIndexEntry elements = idMap_.get(elementId);
1806             if (elements != null) {
1807                 return new ArrayList<>(elements.elements());
1808             }
1809         }
1810         return Collections.emptyList();
1811     }
1812 
1813     /**
1814      * Returns the element with the specified name. If more than one element
1815      * has this name, then this method returns the first one.
1816      *
1817      * @param name the name value to search for
1818      * @param <E> the element type
1819      * @return the element with the specified name
1820      * @throws ElementNotFoundException if no element was found matching the specified name
1821      */
1822     @SuppressWarnings("unchecked")
1823     public <E extends DomElement> E getElementByName(final String name) throws ElementNotFoundException {
1824         if (name != null) {
1825             ensureMappedElementsBuilt();
1826             final MappedElementIndexEntry elements = nameMap_.get(name);
1827             if (elements != null) {
1828                 return (E) elements.first();
1829             }
1830         }
1831         throw new ElementNotFoundException("*", DomElement.NAME_ATTRIBUTE, name);
1832     }
1833 
1834     /**
1835      * Returns the elements with the specified name attribute. If there are no elements
1836      * with the specified name, this method returns an empty list. Please note that
1837      * the lists returned by this method are immutable.
1838      *
1839      * @param name the name value to search for
1840      * @return the elements with the specified name attribute
1841      */
1842     public List<DomElement> getElementsByName(final String name) {
1843         if (name != null) {
1844             ensureMappedElementsBuilt();
1845             final MappedElementIndexEntry elements = nameMap_.get(name);
1846             if (elements != null) {
1847                 return new ArrayList<>(elements.elements());
1848             }
1849         }
1850         return Collections.emptyList();
1851     }
1852 
1853     /**
1854      * Returns the elements with the specified string for their name or ID. If there are
1855      * no elements with the specified name or ID, this method returns an empty list.
1856      *
1857      * @param idAndOrName the value to search for
1858      * @return the elements with the specified string for their name or ID
1859      */
1860     public List<DomElement> getElementsByIdAndOrName(final String idAndOrName) {
1861         if (idAndOrName == null) {
1862             return Collections.emptyList();
1863         }
1864         ensureMappedElementsBuilt();
1865         final MappedElementIndexEntry list1 = idMap_.get(idAndOrName);
1866         final MappedElementIndexEntry list2 = nameMap_.get(idAndOrName);
1867         final List<DomElement> list = new ArrayList<>();
1868         if (list1 != null) {
1869             list.addAll(list1.elements());
1870         }
1871         if (list2 != null) {
1872             for (final DomElement elt : list2.elements()) {
1873                 if (!list.contains(elt)) {
1874                     list.add(elt);
1875                 }
1876             }
1877         }
1878         return list;
1879     }
1880 
1881     /**
1882      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1883      *
1884      * @param node the node that has just been added to the document
1885      */
1886     void notifyNodeAdded(final DomNode node) {
1887         if (node instanceof DomElement element1) {
1888             addMappedElement(element1, true);
1889 
1890             if (node instanceof BaseFrameElement element) {
1891                 frameElements_.add(element);
1892             }
1893 
1894             if (node.getFirstChild() != null) {
1895                 for (final Iterator<HtmlElement> iterator = node.new DescendantHtmlElementsIterator();
1896                         iterator.hasNext();) {
1897                     final HtmlElement child = iterator.next();
1898                     if (child instanceof BaseFrameElement element) {
1899                         frameElements_.add(element);
1900                     }
1901                 }
1902             }
1903 
1904             if ("base".equals(node.getNodeName())) {
1905                 calculateBase();
1906             }
1907         }
1908 
1909         if (node instanceof DomDocumentType doctype && node.getParentNode() == this) {
1910             setDocumentType(doctype);
1911         }
1912         node.onAddedToPage();
1913     }
1914 
1915     /**
1916      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1917      *
1918      * @param node the node that has just been removed from the tree
1919      */
1920     void notifyNodeRemoved(final DomNode node) {
1921         if (node instanceof HtmlElement element) {
1922             removeMappedElement(element, true, true);
1923 
1924             if (node instanceof BaseFrameElement) {
1925                 frameElements_.remove(node);
1926             }
1927             for (final HtmlElement child : node.getHtmlElementDescendants()) {
1928                 if (child instanceof BaseFrameElement) {
1929                     frameElements_.remove(child);
1930                 }
1931             }
1932 
1933             if ("base".equals(node.getNodeName())) {
1934                 calculateBase();
1935             }
1936             return;
1937         }
1938 
1939         if (node instanceof DomDocumentType && node == getDoctype()) {
1940             setDocumentType(null);
1941         }
1942     }
1943 
1944     /**
1945      * Adds an element to the ID and name maps, if necessary.
1946      * @param element the element to be added to the ID and name maps
1947      * @param recurse indicates if children must be added too
1948      */
1949     void addMappedElement(final DomElement element, final boolean recurse) {
1950         // Index is built lazily; skip while not built. ensureMappedElementsBuilt()
1951         // walks the tree once and populates everything on first read.
1952         if (!mappedElementsBuilt_) {
1953             return;
1954         }
1955         if (isAncestorOf(element)) {
1956             addElement(element, recurse);
1957         }
1958     }
1959 
1960     private void ensureMappedElementsBuilt() {
1961         if (mappedElementsBuilt_) {
1962             return;
1963         }
1964 
1965         final DomElement root = getDocumentElement();
1966         if (root != null) {
1967             addElement(root, true);
1968         }
1969 
1970         // Flip the flag only after the maps are populated, so a partial
1971         // failure mid-walk leaves us with built_=false and the next read
1972         // tries again rather than seeing a half-populated index.
1973         mappedElementsBuilt_ = true;
1974     }
1975 
1976     private void addElement(final DomElement element, final boolean recurse) {
1977         final String idValue = element.getAttribute(DomElement.ID_ATTRIBUTE);
1978         if (ATTRIBUTE_NOT_DEFINED != idValue) {
1979             MappedElementIndexEntry elements = idMap_.get(idValue);
1980             if (elements == null) {
1981                 elements = new MappedElementIndexEntry();
1982                 elements.add(element);
1983                 idMap_.put(idValue, elements);
1984             }
1985             else {
1986                 elements.add(element);
1987             }
1988         }
1989 
1990         final String nameValue = element.getAttribute(DomElement.NAME_ATTRIBUTE);
1991         if (ATTRIBUTE_NOT_DEFINED != nameValue) {
1992             MappedElementIndexEntry elements = nameMap_.get(nameValue);
1993             if (elements == null) {
1994                 elements = new MappedElementIndexEntry();
1995                 elements.add(element);
1996                 nameMap_.put(nameValue, elements);
1997             }
1998             else {
1999                 elements.add(element);
2000             }
2001         }
2002 
2003         if (recurse) {
2004             // poor man's approach - we don't use getChildElements()
2005             // to avoid a bunch of object constructions
2006             DomNode nextChild = element.getFirstChild();
2007             while (nextChild != null) {
2008                 if (nextChild instanceof DomElement domElement) {
2009                     addElement(domElement, true);
2010                 }
2011                 nextChild = nextChild.getNextSibling();
2012             }
2013         }
2014     }
2015 
2016     /**
2017      * Removes an element and optionally its children from the ID and name maps, if necessary.
2018      * @param element the element to be removed from the ID and name maps
2019      * @param recurse indicates if children must be removed too
2020      * @param descendant indicates of the element was descendant of this HtmlPage, but now its parent might be null
2021      */
2022     void removeMappedElement(final DomElement element, final boolean recurse, final boolean descendant) {
2023         // see addMappedElement: while the index is unbuilt, removals are also no-ops.
2024         if (!mappedElementsBuilt_) {
2025             return;
2026         }
2027         if (descendant || isAncestorOf(element)) {
2028             removeElement(element, recurse);
2029         }
2030     }
2031 
2032     private void removeElement(final DomElement element, final boolean recurse) {
2033         final String idValue = element.getAttribute(DomElement.ID_ATTRIBUTE);
2034         if (ATTRIBUTE_NOT_DEFINED != idValue) {
2035             final MappedElementIndexEntry elements = idMap_.remove(idValue);
2036             if (elements != null) {
2037                 elements.remove(element);
2038                 if (!elements.elements_.isEmpty()) {
2039                     idMap_.put(idValue, elements);
2040                 }
2041             }
2042         }
2043 
2044         final String nameValue = element.getAttribute(DomElement.NAME_ATTRIBUTE);
2045         if (ATTRIBUTE_NOT_DEFINED != nameValue) {
2046             final MappedElementIndexEntry elements = nameMap_.remove(nameValue);
2047             if (elements != null) {
2048                 elements.remove(element);
2049                 if (!elements.elements_.isEmpty()) {
2050                     nameMap_.put(nameValue, elements);
2051                 }
2052             }
2053         }
2054 
2055         if (recurse) {
2056             for (final DomElement child : element.getChildElements()) {
2057                 removeElement(child, true);
2058             }
2059         }
2060     }
2061 
2062     /**
2063      * Indicates if the attribute name indicates that the owning element is mapped.
2064      * @param document the owning document
2065      * @param attributeName the name of the attribute to consider
2066      * @return {@code true} if the owning element should be mapped in its owning page
2067      */
2068     static boolean isMappedElement(final Document document, final String attributeName) {
2069         return document instanceof HtmlPage
2070             && (DomElement.NAME_ATTRIBUTE.equals(attributeName) || DomElement.ID_ATTRIBUTE.equals(attributeName));
2071     }
2072 
2073     private void calculateBase() {
2074         final List<HtmlElement> baseElements = getDocumentElement().getStaticElementsByTagName("base");
2075 
2076         base_ = null;
2077         for (final HtmlElement baseElement : baseElements) {
2078             if (baseElement instanceof HtmlBase base) {
2079                 if (base_ != null) {
2080                     notifyIncorrectness("Multiple 'base' detected, only the first is used.");
2081                     break;
2082                 }
2083                 base_ = base;
2084             }
2085         }
2086     }
2087 
2088     /**
2089      * Loads the content of the contained frames. This is done after the page is completely loaded, to allow script
2090      * contained in the frames to reference elements from the page located after the closing &lt;/frame&gt; tag.
2091      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
2092      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to {@code true}
2093      */
2094     void loadFrames() throws FailingHttpStatusCodeException {
2095         for (final BaseFrameElement frameElement : new ArrayList<>(frameElements_)) {
2096             // test if the frame should really be loaded:
2097             // if a script has already changed its content, it should be skipped
2098             // use == and not equals(...) to identify initial content (versus URL set to "about:blank")
2099             if (frameElement.getEnclosedWindow() != null
2100                     && UrlUtils.URL_ABOUT_BLANK == frameElement.getEnclosedPage().getUrl()
2101                     && !frameElement.isContentLoaded()) {
2102                 frameElement.loadInnerPage();
2103             }
2104         }
2105     }
2106 
2107     /**
2108      * Gives a basic representation for debugging purposes.
2109      * @return a basic representation
2110      */
2111     @Override
2112     public String toString() {
2113         final StringBuilder builder = new StringBuilder()
2114             .append("HtmlPage(")
2115             .append(getUrl())
2116             .append(")@")
2117             .append(hashCode());
2118         return builder.toString();
2119     }
2120 
2121     /**
2122      * Gets the meta tag for a given {@code http-equiv} value.
2123      * @param httpEquiv the {@code http-equiv} value
2124      * @return a list of {@link HtmlMeta}
2125      */
2126     protected List<HtmlMeta> getMetaTags(final String httpEquiv) {
2127         if (getDocumentElement() == null) {
2128             return Collections.emptyList(); // weird case, for instance if document.documentElement has been removed
2129         }
2130         final List<HtmlMeta> tags = getDocumentElement().getStaticElementsByTagName("meta");
2131         final List<HtmlMeta> foundTags = new ArrayList<>();
2132         for (final HtmlMeta htmlMeta : tags) {
2133             if (httpEquiv.equalsIgnoreCase(htmlMeta.getHttpEquivAttribute())) {
2134                 foundTags.add(htmlMeta);
2135             }
2136         }
2137         return foundTags;
2138     }
2139 
2140     /**
2141      * Creates a clone of this instance, and clears cached state to be not shared with the original.
2142      *
2143      * @return a clone of this instance
2144      */
2145     @Override
2146     protected HtmlPage clone() {
2147         final HtmlPage result = (HtmlPage) super.clone();
2148         result.elementWithFocus_ = null;
2149 
2150         result.idMap_ = new ConcurrentHashMap<>();
2151         result.nameMap_ = new ConcurrentHashMap<>();
2152         result.mappedElementsBuilt_ = false;
2153 
2154         return result;
2155     }
2156 
2157     /**
2158      * {@inheritDoc}
2159      */
2160     @Override
2161     public HtmlPage cloneNode(final boolean deep) {
2162         // we need the ScriptObject clone before cloning the kids.
2163         final HtmlPage result = (HtmlPage) super.cloneNode(false);
2164         if (getWebClient().isJavaScriptEnabled()) {
2165             final HtmlUnitScriptable jsObjClone = getScriptableObject().clone();
2166             jsObjClone.setDomNode(result);
2167         }
2168 
2169         // if deep, clone the kids too, and re initialize parts of the clone
2170         if (deep) {
2171             // this was previously synchronized but that makes not sense, why
2172             // lock the source against a copy only one has a reference too,
2173             // because result is a local reference
2174             result.attributeListeners_ = null;
2175 
2176             result.selectionRanges_ = new ArrayList<>(3);
2177             // the original one is synchronized so we should do that here too, shouldn't we?
2178             result.afterLoadActions_ = Collections.synchronizedList(new ArrayList<>());
2179             result.frameElements_ = new ArrayList<>();
2180             for (DomNode child = getFirstChild(); child != null; child = child.getNextSibling()) {
2181                 result.appendChild(child.cloneNode(true));
2182             }
2183         }
2184         return result;
2185     }
2186 
2187     /**
2188      * Adds an HtmlAttributeChangeListener to the listener list.
2189      * The listener is registered for all attributes of all HtmlElements contained in this page.
2190      *
2191      * @param listener the attribute change listener to be added
2192      * @see #removeHtmlAttributeChangeListener(HtmlAttributeChangeListener)
2193      */
2194     public void addHtmlAttributeChangeListener(final HtmlAttributeChangeListener listener) {
2195         WebAssert.notNull("listener", listener);
2196         synchronized (lock_) {
2197             if (attributeListeners_ == null) {
2198                 attributeListeners_ = new LinkedHashSet<>();
2199             }
2200             attributeListeners_.add(listener);
2201         }
2202     }
2203 
2204     /**
2205      * Removes an HtmlAttributeChangeListener from the listener list.
2206      * This method should be used to remove HtmlAttributeChangeListener that were registered
2207      * for all attributes of all HtmlElements contained in this page.
2208      *
2209      * @param listener the attribute change listener to be removed
2210      * @see #addHtmlAttributeChangeListener(HtmlAttributeChangeListener)
2211      */
2212     public void removeHtmlAttributeChangeListener(final HtmlAttributeChangeListener listener) {
2213         WebAssert.notNull("listener", listener);
2214         synchronized (lock_) {
2215             if (attributeListeners_ != null) {
2216                 attributeListeners_.remove(listener);
2217             }
2218         }
2219     }
2220 
2221     /**
2222      * Notifies all registered listeners for the given event to add an attribute.
2223      * @param event the event to fire
2224      */
2225     void fireHtmlAttributeAdded(final HtmlAttributeChangeEvent event) {
2226         final List<HtmlAttributeChangeListener> listeners = safeGetAttributeListeners();
2227         if (listeners != null) {
2228             for (final HtmlAttributeChangeListener listener : listeners) {
2229                 listener.attributeAdded(event);
2230             }
2231         }
2232     }
2233 
2234     /**
2235      * Notifies all registered listeners for the given event to replace an attribute.
2236      * @param event the event to fire
2237      */
2238     void fireHtmlAttributeReplaced(final HtmlAttributeChangeEvent event) {
2239         final List<HtmlAttributeChangeListener> listeners = safeGetAttributeListeners();
2240         if (listeners != null) {
2241             for (final HtmlAttributeChangeListener listener : listeners) {
2242                 listener.attributeReplaced(event);
2243             }
2244         }
2245     }
2246 
2247     /**
2248      * Notifies all registered listeners for the given event to remove an attribute.
2249      * @param event the event to fire
2250      */
2251     void fireHtmlAttributeRemoved(final HtmlAttributeChangeEvent event) {
2252         final List<HtmlAttributeChangeListener> listeners = safeGetAttributeListeners();
2253         if (listeners != null) {
2254             for (final HtmlAttributeChangeListener listener : listeners) {
2255                 listener.attributeRemoved(event);
2256             }
2257         }
2258     }
2259 
2260     private List<HtmlAttributeChangeListener> safeGetAttributeListeners() {
2261         synchronized (lock_) {
2262             if (attributeListeners_ != null) {
2263                 return new ArrayList<>(attributeListeners_);
2264             }
2265             return null;
2266         }
2267     }
2268 
2269     /**
2270      * {@inheritDoc}
2271      */
2272     @Override
2273     protected void checkChildHierarchy(final org.w3c.dom.Node newChild) throws DOMException {
2274         if (newChild instanceof Element) {
2275             if (getDocumentElement() != null) {
2276                 throw new DOMException(DOMException.HIERARCHY_REQUEST_ERR,
2277                     "The Document may only have a single child Element.");
2278             }
2279         }
2280         else if (newChild instanceof DocumentType) {
2281             if (getDoctype() != null) {
2282                 throw new DOMException(DOMException.HIERARCHY_REQUEST_ERR,
2283                     "The Document may only have a single child DocumentType.");
2284             }
2285         }
2286         else if (!(newChild instanceof Comment || newChild instanceof ProcessingInstruction)) {
2287             throw new DOMException(DOMException.HIERARCHY_REQUEST_ERR,
2288                 "The Document may not have a child of this type: " + newChild.getNodeType());
2289         }
2290         super.checkChildHierarchy(newChild);
2291     }
2292 
2293     /**
2294      * Returns {@code true} if an HTML parser is operating on this page, adding content to it.
2295      * @return {@code true} if an HTML parser is operating on this page, adding content to it
2296      */
2297     public boolean isBeingParsed() {
2298         return parserCount_ > 0;
2299     }
2300 
2301     /**
2302      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2303      *
2304      * Called by the HTML parser to let the page know that it has started parsing some content for this page.
2305      */
2306     public void registerParsingStart() {
2307         parserCount_++;
2308     }
2309 
2310     /**
2311      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2312      *
2313      * Called by the HTML parser to let the page know that it has finished parsing some content for this page.
2314      */
2315     public void registerParsingEnd() {
2316         parserCount_--;
2317     }
2318 
2319     /**
2320      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2321      *
2322      * Returns {@code true} if an HTML parser is parsing a non-inline HTML snippet to add content
2323      * to this page. Non-inline content is content that is parsed for the page, but not in the
2324      * same stream as the page itself -- basically anything other than <code>document.write()</code>
2325      * or <code>document.writeln()</code>: <code>innerHTML</code>, <code>outerHTML</code>,
2326      * <code>document.createElement()</code>, etc.
2327      *
2328      * @return {@code true} if an HTML parser is parsing a non-inline HTML snippet to add content
2329      *         to this page
2330      */
2331     public boolean isParsingHtmlSnippet() {
2332         return snippetParserCount_ > 0;
2333     }
2334 
2335     /**
2336      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2337      *
2338      * Called by the HTML parser to let the page know that it has started parsing a non-inline HTML snippet.
2339      */
2340     public void registerSnippetParsingStart() {
2341         snippetParserCount_++;
2342     }
2343 
2344     /**
2345      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2346      *
2347      * Called by the HTML parser to let the page know that it has finished parsing a non-inline HTML snippet.
2348      */
2349     public void registerSnippetParsingEnd() {
2350         snippetParserCount_--;
2351     }
2352 
2353     /**
2354      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2355      *
2356      * Returns {@code true} if an HTML parser is parsing an inline HTML snippet to add content
2357      * to this page. Inline content is content inserted into the parser stream dynamically
2358      * while the page is being parsed (i.e. <code>document.write()</code> or <code>document.writeln()</code>).
2359      *
2360      * @return {@code true} if an HTML parser is parsing an inline HTML snippet to add content
2361      *         to this page
2362      */
2363     public boolean isParsingInlineHtmlSnippet() {
2364         return inlineSnippetParserCount_ > 0;
2365     }
2366 
2367     /**
2368      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2369      *
2370      * Called by the HTML parser to let the page know that it has started parsing an inline HTML snippet.
2371      */
2372     public void registerInlineSnippetParsingStart() {
2373         inlineSnippetParserCount_++;
2374     }
2375 
2376     /**
2377      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2378      *
2379      * Called by the HTML parser to let the page know that it has finished parsing an inline HTML snippet.
2380      */
2381     public void registerInlineSnippetParsingEnd() {
2382         inlineSnippetParserCount_--;
2383     }
2384 
2385     /**
2386      * Refreshes the page by sending the same parameters as previously sent to get this page.
2387      * @return the newly loaded page.
2388      * @throws IOException if an IO problem occurs
2389      */
2390     public Page refresh() throws IOException {
2391         return getWebClient().getPage(getWebResponse().getWebRequest());
2392     }
2393 
2394     /**
2395      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2396      * <p>
2397      * Parses the given string as would it belong to the content being parsed
2398      * at the current parsing position
2399      * </p>
2400      * @param string the HTML code to write in place
2401      */
2402     public void writeInParsedStream(final String string) {
2403         getDOMBuilder().pushInputString(string);
2404     }
2405 
2406     /**
2407      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2408      *
2409      * Sets the builder to allow page to send content from document.write(ln) calls.
2410      * @param htmlUnitDOMBuilder the builder
2411      */
2412     public void setDOMBuilder(final HTMLParserDOMBuilder htmlUnitDOMBuilder) {
2413         domBuilder_ = htmlUnitDOMBuilder;
2414     }
2415 
2416     /**
2417      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2418      *
2419      * Returns the current builder.
2420      * @return the current builder
2421      */
2422     public HTMLParserDOMBuilder getDOMBuilder() {
2423         return domBuilder_;
2424     }
2425 
2426     /**
2427      * <p>Returns all namespaces defined in the root element of this page.</p>
2428      * <p>The default namespace has a key of an empty string.</p>
2429      * @return all namespaces defined in the root element of this page
2430      */
2431     public Map<String, String> getNamespaces() {
2432         final org.w3c.dom.NamedNodeMap attributes = getDocumentElement().getAttributes();
2433         final Map<String, String> namespaces = new HashMap<>();
2434         for (int i = 0; i < attributes.getLength(); i++) {
2435             final Attr attr = (Attr) attributes.item(i);
2436             String name = attr.getName();
2437             if (name.startsWith("xmlns")) {
2438                 int startPos = 5;
2439                 if (name.length() > 5 && name.charAt(5) == ':') {
2440                     startPos = 6;
2441                 }
2442                 name = name.substring(startPos);
2443                 namespaces.put(name, attr.getValue());
2444             }
2445         }
2446         return namespaces;
2447     }
2448 
2449     /**
2450      * {@inheritDoc}
2451      */
2452     @Override
2453     public void setDocumentType(final DocumentType type) {
2454         super.setDocumentType(type);
2455     }
2456 
2457     /**
2458      * Saves the current page, with all images, to the specified location.
2459      * The default behavior removes all script elements.
2460      *
2461      * @param file file to write this page into
2462      * @throws IOException If an error occurs
2463      */
2464     public void save(final File file) throws IOException {
2465         new XmlSerializer().save(this, file);
2466     }
2467 
2468     /**
2469      * Returns whether the current page mode is in {@code quirks mode} or in {@code standards mode}.
2470      * @return true for {@code quirks mode}, false for {@code standards mode}
2471      */
2472     public boolean isQuirksMode() {
2473         return "BackCompat".equals(((HTMLDocument) getScriptableObject()).getCompatMode());
2474     }
2475 
2476     /**
2477      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2478      * {@inheritDoc}
2479      */
2480     @Override
2481     public boolean isAttachedToPage() {
2482         return true;
2483     }
2484 
2485     /**
2486      * {@inheritDoc}
2487      */
2488     @Override
2489     public boolean isHtmlPage() {
2490         return true;
2491     }
2492 
2493     /**
2494      * The base URL used to resolve relative URLs.
2495      * @return the base URL
2496      */
2497     public URL getBaseURL() {
2498         URL baseUrl;
2499         if (base_ == null) {
2500             baseUrl = getUrl();
2501             final WebWindow window = getEnclosingWindow();
2502             final boolean frame = window != null && window != window.getTopWindow();
2503             if (frame) {
2504                 final boolean frameSrcIsNotSet = baseUrl == UrlUtils.URL_ABOUT_BLANK;
2505                 final boolean frameSrcIsJs = "javascript".equals(baseUrl.getProtocol());
2506                 if (frameSrcIsNotSet || frameSrcIsJs) {
2507                     baseUrl = window.getTopWindow().getEnclosedPage().getWebResponse()
2508                         .getWebRequest().getUrl();
2509                 }
2510             }
2511             else if (baseUrl_ != null) {
2512                 baseUrl = baseUrl_;
2513             }
2514         }
2515         else {
2516             final String href = base_.getHrefAttribute().trim();
2517             if (org.htmlunit.util.StringUtils.isEmptyOrNull(href)) {
2518                 baseUrl = getUrl();
2519             }
2520             else {
2521                 final URL url = getUrl();
2522                 try {
2523                     if (href.startsWith("http://") || href.startsWith("https://")) {
2524                         baseUrl = new URL(href);
2525                     }
2526                     else if (href.startsWith("//")) {
2527                         baseUrl = new URL("%s:%s".formatted(url.getProtocol(), href));
2528                     }
2529                     else if (href.length() > 0 && href.charAt(0) == '/') {
2530                         final int port = Window.getPort(url);
2531                         baseUrl = new URL("%s://%s:%d%s".formatted(url.getProtocol(), url.getHost(), port, href));
2532                     }
2533                     else if (url.toString().endsWith("/")) {
2534                         baseUrl = new URL("%s%s".formatted(url, href));
2535                     }
2536                     else {
2537                         baseUrl = new URL(UrlUtils.resolveUrl(url, href));
2538                     }
2539                 }
2540                 catch (final MalformedURLException e) {
2541                     notifyIncorrectness("Invalid base url: \"" + href + "\", ignoring it");
2542                     baseUrl = url;
2543                 }
2544             }
2545         }
2546 
2547         return baseUrl;
2548     }
2549 
2550     /**
2551      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2552      *
2553      * Adds an {@link AutoCloseable}, which would be closed during the {@link #cleanUp()}.
2554      * @param autoCloseable the autoclosable
2555      */
2556     public void addAutoCloseable(final AutoCloseable autoCloseable) {
2557         if (autoCloseable == null) {
2558             return;
2559         }
2560 
2561         if (autoCloseableList_ == null) {
2562             autoCloseableList_ = new ArrayList<>();
2563         }
2564         autoCloseableList_.add(autoCloseable);
2565     }
2566 
2567     /**
2568      * {@inheritDoc}
2569      */
2570     @Override
2571     public boolean handles(final Event event) {
2572         if (Event.TYPE_BLUR.equals(event.getType()) || Event.TYPE_FOCUS.equals(event.getType())) {
2573             return true;
2574         }
2575         return super.handles(event);
2576     }
2577 
2578     /**
2579      * Sets the {@link ElementFromPointHandler}.
2580      * @param elementFromPointHandler the handler
2581      */
2582     public void setElementFromPointHandler(final ElementFromPointHandler elementFromPointHandler) {
2583         elementFromPointHandler_ = elementFromPointHandler;
2584     }
2585 
2586     /**
2587      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2588      *
2589      * Returns the element for the specified x coordinate and the specified y coordinate.
2590      *
2591      * @param x the x offset, in pixels
2592      * @param y the y offset, in pixels
2593      * @return the element for the specified x coordinate and the specified y coordinate
2594      */
2595     public HtmlElement getElementFromPoint(final int x, final int y) {
2596         if (elementFromPointHandler_ == null) {
2597             if (LOG.isWarnEnabled()) {
2598                 LOG.warn("ElementFromPointHandler was not specicifed for " + this);
2599             }
2600             if (x <= 0 || y <= 0) {
2601                 return null;
2602             }
2603             return getBody();
2604         }
2605         return elementFromPointHandler_.getElementFromPoint(this, x, y);
2606     }
2607 
2608     /**
2609      * Moves the focus to the specified element. This will trigger any relevant JavaScript
2610      * event handlers.
2611      *
2612      * @param newElement the element that will receive the focus, use {@code null} to remove focus from any element
2613      * @return true if the specified element now has the focus
2614      * @see #getFocusedElement()
2615      */
2616     public boolean setFocusedElement(final DomElement newElement) {
2617         return setFocusedElement(newElement, false);
2618     }
2619 
2620     /**
2621      * Moves the focus to the specified element. This will trigger any relevant JavaScript
2622      * event handlers.
2623      *
2624      * @param newElement the element that will receive the focus, use {@code null} to remove focus from any element
2625      * @param windowActivated - whether the enclosing window got focus resulting in specified element getting focus
2626      * @return true if the specified element now has the focus
2627      * @see #getFocusedElement()
2628      */
2629     public boolean setFocusedElement(final DomElement newElement, final boolean windowActivated) {
2630         if (elementWithFocus_ == newElement && !windowActivated) {
2631             // nothing to do
2632             return true;
2633         }
2634 
2635         final DomElement oldFocusedElement = elementWithFocus_;
2636         elementWithFocus_ = null;
2637 
2638         if (!windowActivated) {
2639             if (oldFocusedElement != null) {
2640                 oldFocusedElement.removeFocus();
2641                 oldFocusedElement.fireEvent(Event.TYPE_BLUR);
2642 
2643                 oldFocusedElement.fireEvent(Event.TYPE_FOCUS_OUT);
2644             }
2645         }
2646 
2647         elementWithFocus_ = newElement;
2648 
2649         // use newElement in the code below because element elementWithFocus_
2650         // might be changed by another thread
2651         if (newElement != null) {
2652             newElement.focus();
2653             newElement.fireEvent(Event.TYPE_FOCUS);
2654 
2655             newElement.fireEvent(Event.TYPE_FOCUS_IN);
2656         }
2657 
2658         // If a page reload happened as a result of the focus change then obviously this
2659         // element will not have the focus because its page has gone away.
2660         return this == getEnclosingWindow().getEnclosedPage();
2661     }
2662 
2663     /**
2664      * Returns the element with the focus or null if no element has the focus.
2665      * @return the element with focus or null
2666      * @see #setFocusedElement(DomElement)
2667      */
2668     public DomElement getFocusedElement() {
2669         return elementWithFocus_;
2670     }
2671 
2672     /**
2673      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2674      *
2675      * Sets the element with focus.
2676      * @param elementWithFocus the element with focus
2677      */
2678     public void setElementWithFocus(final DomElement elementWithFocus) {
2679         elementWithFocus_ = elementWithFocus;
2680     }
2681 
2682     /**
2683      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2684      * Returns the currently active element.
2685      *
2686      * @return the element that currently has focus, or the document's
2687      *         {@code body} element if no element has focus, or {@code null}
2688      *         if the document has no {@code body} element
2689      */
2690     public HtmlElement getActiveElement() {
2691         final DomElement activeElement = getFocusedElement();
2692         if (activeElement instanceof HtmlElement element) {
2693             return element;
2694         }
2695 
2696         final HtmlElement body = getBody();
2697         if (body != null) {
2698             return body;
2699         }
2700         return null;
2701     }
2702 
2703     /**
2704      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2705      *
2706      * <p>Returns the page's current selection ranges.</p>
2707      *
2708      * @return the page's current selection ranges
2709      */
2710     public List<SimpleRange> getSelectionRanges() {
2711         return selectionRanges_;
2712     }
2713 
2714     /**
2715      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2716      *
2717      * <p>Makes the specified selection range the *only* selection range on this page.</p>
2718      *
2719      * @param selectionRange the selection range
2720      */
2721     public void setSelectionRange(final SimpleRange selectionRange) {
2722         selectionRanges_.clear();
2723         selectionRanges_.add(selectionRange);
2724     }
2725 
2726     /**
2727      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2728      *
2729      * Execute a Function in the given context.
2730      *
2731      * @param function the JavaScript Function to call
2732      * @param thisObject the "this" object to be used during invocation
2733      * @param args the arguments to pass into the call
2734      * @param htmlElement the HTML element for which this script is being executed
2735      *        This element will be the context during the JavaScript execution. If null,
2736      *        the context will default to the page.
2737      * @return a ScriptResult which will contain both the current page (which may be different from
2738      *        the previous page) and a JavaScript result object.
2739      */
2740     public ScriptResult executeJavaScriptFunction(final Object function, final Object thisObject,
2741             final Object[] args, final DomNode htmlElement) {
2742         if (!getWebClient().isJavaScriptEnabled()) {
2743             return new ScriptResult(null);
2744         }
2745 
2746         final JavaScriptEngine engine = (JavaScriptEngine) getWebClient().getJavaScriptEngine();
2747         final Object result = engine.callFunction(this,
2748                                 (Function) function, (Scriptable) thisObject, args, htmlElement);
2749 
2750         return new ScriptResult(result);
2751     }
2752 
2753     private void writeObject(final ObjectOutputStream oos) throws IOException {
2754         oos.defaultWriteObject();
2755         oos.writeObject(originalCharset_ == null ? null : originalCharset_.name());
2756     }
2757 
2758     private void readObject(final ObjectInputStream ois) throws ClassNotFoundException, IOException {
2759         ois.defaultReadObject();
2760         final String charsetName = (String) ois.readObject();
2761         if (charsetName != null) {
2762             originalCharset_ = Charset.forName(charsetName);
2763         }
2764     }
2765 
2766     /**
2767      * {@inheritDoc}
2768      */
2769     @Override
2770     public void setNodeValue(final String value) {
2771         // Default behavior is to do nothing, overridden in some subclasses
2772     }
2773 
2774     /**
2775      * {@inheritDoc}
2776      */
2777     @Override
2778     public void setPrefix(final String prefix) {
2779         // Empty.
2780     }
2781 
2782     /**
2783      * {@inheritDoc}
2784      */
2785     @Override
2786     public void clearComputedStyles() {
2787         if (computedStylesCache_ != null) {
2788             computedStylesCache_.clear();
2789         }
2790     }
2791 
2792     /**
2793      * {@inheritDoc}
2794      */
2795     @Override
2796     public void clearComputedStyles(final DomElement element) {
2797         if (computedStylesCache_ != null) {
2798             computedStylesCache_.remove(element);
2799         }
2800     }
2801 
2802     /**
2803      * {@inheritDoc}
2804      */
2805     @Override
2806     public void clearComputedStylesUpToRoot(final DomElement element) {
2807         if (computedStylesCache_ != null) {
2808             computedStylesCache_.remove(element);
2809 
2810             DomNode parent = element.getParentNode();
2811             while (parent != null) {
2812                 computedStylesCache_.remove(parent);
2813                 parent = parent.getParentNode();
2814             }
2815         }
2816     }
2817 
2818     /**
2819      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2820      *
2821      * @param element the element to clear its cache
2822      * @param normalizedPseudo the pseudo attribute
2823      * @return the cached ComputedCssStyleDeclaration object or null
2824      */
2825     public ComputedCssStyleDeclaration getStyleFromCache(final DomElement element,
2826             final String normalizedPseudo) {
2827         return getCssPropertiesCache().get(element, normalizedPseudo);
2828     }
2829 
2830     /**
2831      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2832      *
2833      * Caches a ComputedCssStyleDeclaration object.
2834      * @param element the element to clear its cache
2835      * @param normalizedPseudo the pseudo attribute
2836      * @param style the ComputedCssStyleDeclaration to cache
2837      */
2838     public void putStyleIntoCache(final DomElement element, final String normalizedPseudo,
2839             final ComputedCssStyleDeclaration style) {
2840         getCssPropertiesCache().put(element, normalizedPseudo, style);
2841     }
2842 
2843     /**
2844      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2845      *
2846      * @return a list of all styles from this page (&lt;style&gt; and &lt;link rel=stylesheet&gt;).
2847      *         This returns an empty list if css support is disabled in the web client options.
2848      */
2849     public List<CssStyleSheet> getStyleSheets() {
2850         final List<CssStyleSheet> styles = new ArrayList<>();
2851         if (getWebClient().getOptions().isCssEnabled()) {
2852             for (final HtmlElement htmlElement : getHtmlElementDescendants()) {
2853                 if (htmlElement instanceof HtmlStyle style) {
2854                     styles.add(style.getSheet());
2855                     continue;
2856                 }
2857 
2858                 if (htmlElement instanceof HtmlLink link) {
2859                     if (link.isStyleSheetLink()) {
2860                         styles.add(link.getSheet());
2861                     }
2862                 }
2863             }
2864         }
2865         return styles;
2866     }
2867 
2868     /**
2869      * Returns the computed styles cache for this page.
2870      *
2871      * @return the computed styles cache for this page
2872      */
2873     private ComputedStylesCache getCssPropertiesCache() {
2874         if (computedStylesCache_ == null) {
2875             computedStylesCache_ = new ComputedStylesCache();
2876 
2877             // maintain the style cache
2878             final DomHtmlAttributeChangeListenerImpl listener = new DomHtmlAttributeChangeListenerImpl();
2879             addDomChangeListener(listener);
2880             addHtmlAttributeChangeListener(listener);
2881         }
2882         return computedStylesCache_;
2883     }
2884 
2885     /**
2886      * <p>Listens for changes anywhere in the document and evicts cached computed styles whenever something relevant
2887      * changes. Note that the very lazy way of doing this (completely clearing the cache every time something happens)
2888      * results in very meager performance gains. In order to get good (but still correct) performance, we need to be
2889      * a little smarter.</p>
2890      *
2891      * <p>CSS 2.1 has the following <a href="http://www.w3.org/TR/CSS21/selector.html">selector types</a> (where "SN" is
2892      * shorthand for "the selected node"):</p>
2893      *
2894      * <ol>
2895      *   <li><em>Universal</em> (i.e. "*"): Affected by the removal of SN from the document.</li>
2896      *   <li><em>Type</em> (i.e. "div"): Affected by the removal of SN from the document.</li>
2897      *   <li><em>Descendant</em> (i.e. "div span"): Affected by changes to SN or to any of its ancestors.</li>
2898      *   <li><em>Child</em> (i.e. "div &gt; span"): Affected by changes to SN or to its parent.</li>
2899      *   <li><em>Adjacent Sibling</em> (i.e. "table + p"): Affected by changes to SN or its previous sibling.</li>
2900      *   <li><em>Attribute</em> (i.e. "div.up, div[class~=up]"): Affected by changes to an attribute of SN.</li>
2901      *   <li><em>ID</em> (i.e. "#header"): Affected by changes to the <code>id</code> attribute of SN.</li>
2902      *   <li><em>Pseudo-Elements and Pseudo-Classes</em> (i.e. "p:first-child"): Affected by changes to parent.</li>
2903      * </ol>
2904      *
2905      * <p>Together, these rules dictate that the smart (but still lazy) way of removing elements from the computed style
2906      * cache is as follows -- whenever a node changes in any way, the cache needs to be cleared of styles for nodes
2907      * which:</p>
2908      *
2909      * <ul>
2910      *   <li>are actually the same node as the node that changed</li>
2911      *   <li>are siblings of the node that changed</li>
2912      *   <li>are descendants of the node that changed</li>
2913      * </ul>
2914      *
2915      * <p>Additionally, whenever a <code>style</code> node or a <code>link</code> node
2916      * with <code>rel=stylesheet</code> is added or
2917      * removed, all elements should be removed from the computed style cache.</p>
2918      */
2919     private class DomHtmlAttributeChangeListenerImpl implements DomChangeListener, HtmlAttributeChangeListener {
2920 
2921         /**
2922          * Ctor.
2923          */
2924         DomHtmlAttributeChangeListenerImpl() {
2925             super();
2926         }
2927 
2928         /**
2929          * {@inheritDoc}
2930          */
2931         @Override
2932         public void nodeAdded(final DomChangeEvent event) {
2933             nodeChanged(event.getChangedNode(), null);
2934         }
2935 
2936         /**
2937          * {@inheritDoc}
2938          */
2939         @Override
2940         public void nodeDeleted(final DomChangeEvent event) {
2941             nodeChanged(event.getChangedNode(), null);
2942         }
2943 
2944         /**
2945          * {@inheritDoc}
2946          */
2947         @Override
2948         public void attributeAdded(final HtmlAttributeChangeEvent event) {
2949             nodeChanged(event.getHtmlElement(), event.getName());
2950         }
2951 
2952         /**
2953          * {@inheritDoc}
2954          */
2955         @Override
2956         public void attributeRemoved(final HtmlAttributeChangeEvent event) {
2957             nodeChanged(event.getHtmlElement(), event.getName());
2958         }
2959 
2960         /**
2961          * {@inheritDoc}
2962          */
2963         @Override
2964         public void attributeReplaced(final HtmlAttributeChangeEvent event) {
2965             nodeChanged(event.getHtmlElement(), event.getName());
2966         }
2967 
2968         private void nodeChanged(final DomNode changedNode, final String attribName) {
2969             // If a stylesheet was changed, all of our calculations could be off; clear the cache.
2970             if (changedNode instanceof HtmlStyle) {
2971                 clearComputedStyles();
2972                 return;
2973             }
2974             if (changedNode instanceof HtmlLink link) {
2975                 if (link.isStyleSheetLink()) {
2976                     clearComputedStyles();
2977                     return;
2978                 }
2979             }
2980 
2981             // Apparently it wasn't a stylesheet that changed; be semi-smart about what we evict and when.
2982             // null means that a node was added/removed; we always have to take care of this for the parents
2983             final boolean clearParents = attribName == null || ATTRIBUTES_AFFECTING_PARENT.contains(attribName);
2984             if (computedStylesCache_ != null) {
2985                 computedStylesCache_.nodeChanged(changedNode, clearParents);
2986             }
2987         }
2988     }
2989 
2990     /**
2991      * Cache computed styles when possible, because their calculation is very expensive.
2992      * We use a weak hash map because we don't want this cache to be the only reason
2993      * nodes are kept around in the JVM, if all other references to them are gone.
2994      */
2995     private static final class ComputedStylesCache implements Serializable {
2996         private transient WeakHashMap<DomElement, Map<String, ComputedCssStyleDeclaration>>
2997                     computedStyles_ = new WeakHashMap<>();
2998 
2999         /**
3000          * Ctor.
3001          */
3002         ComputedStylesCache() {
3003             super();
3004         }
3005 
3006         public synchronized ComputedCssStyleDeclaration get(final DomElement element,
3007                 final String normalizedPseudo) {
3008             final Map<String, ComputedCssStyleDeclaration> elementMap = computedStyles_.get(element);
3009             if (elementMap != null) {
3010                 return elementMap.get(normalizedPseudo);
3011             }
3012             return null;
3013         }
3014 
3015         public synchronized void put(final DomElement element,
3016                 final String normalizedPseudo, final ComputedCssStyleDeclaration style) {
3017             final Map<String, ComputedCssStyleDeclaration>
3018                     elementMap = computedStyles_.computeIfAbsent(element, k -> new WeakHashMap<>());
3019             elementMap.put(normalizedPseudo, style);
3020         }
3021 
3022         public synchronized void nodeChanged(final DomNode changed, final boolean clearParents) {
3023             final Iterator<Map.Entry<DomElement, Map<String, ComputedCssStyleDeclaration>>>
3024                     i = computedStyles_.entrySet().iterator();
3025             while (i.hasNext()) {
3026                 final Map.Entry<DomElement, Map<String, ComputedCssStyleDeclaration>> entry = i.next();
3027                 final DomElement node = entry.getKey();
3028                 if (changed == node
3029                     || changed.getParentNode() == node.getParentNode()
3030                     || changed.isAncestorOf(node)
3031                     || clearParents && node.isAncestorOf(changed)) {
3032                     i.remove();
3033                 }
3034             }
3035 
3036             // maybe this is a better solution but I have to think a bit more about this
3037             //
3038             //            if (computedStyles_.isEmpty()) {
3039             //                return;
3040             //            }
3041             //
3042             //            // remove all siblings
3043             //            DomNode parent = changed.getParentNode();
3044             //            if (parent != null) {
3045             //                for (DomNode sibling : parent.getChildNodes()) {
3046             //                    computedStyles_.remove(sibling.getScriptableObject());
3047             //                }
3048             //
3049             //                if (clearParents) {
3050             //                    // remove all parents
3051             //                    while (parent != null) {
3052             //                        computedStyles_.remove(parent.getScriptableObject());
3053             //                        parent = parent.getParentNode();
3054             //                    }
3055             //                }
3056             //            }
3057             //
3058             //            // remove changed itself and all descendants
3059             //            computedStyles_.remove(changed.getScriptableObject());
3060             //            for (DomNode descendant : changed.getDescendants()) {
3061             //                computedStyles_.remove(descendant.getScriptableObject());
3062             //            }
3063         }
3064 
3065         public synchronized void clear() {
3066             computedStyles_.clear();
3067         }
3068 
3069         public synchronized Map<String, ComputedCssStyleDeclaration> remove(final DomNode element) {
3070             return computedStyles_.remove(element);
3071         }
3072 
3073         private void readObject(final ObjectInputStream in) throws IOException, ClassNotFoundException {
3074             in.defaultReadObject();
3075             computedStyles_ = new WeakHashMap<>();
3076         }
3077     }
3078 
3079     private static final class MappedElementIndexEntry implements Serializable {
3080         private final ArrayList<DomElement> elements_;
3081         private boolean sorted_;
3082 
3083         MappedElementIndexEntry() {
3084             // we do not expect to many elements having the same id/name
3085             elements_ = new ArrayList<>(2);
3086             sorted_ = true;
3087         }
3088 
3089         void add(final DomElement element) {
3090             if (elements_.indexOf(element) == -1) {
3091                 elements_.add(element);
3092                 sorted_ = elements_.size() < 2;
3093             }
3094         }
3095 
3096         DomElement first() {
3097             if (elements_.isEmpty()) {
3098                 return null;
3099             }
3100 
3101             if (sorted_) {
3102                 return elements_.get(0);
3103             }
3104 
3105             elements_.sort(DOCUMENT_POSITION_COMPERATOR);
3106             sorted_ = true;
3107 
3108             return elements_.get(0);
3109         }
3110 
3111         List<DomElement> elements() {
3112             if (sorted_) {
3113                 return elements_;
3114             }
3115 
3116             elements_.sort(DOCUMENT_POSITION_COMPERATOR);
3117             sorted_ = true;
3118 
3119             return elements_;
3120         }
3121 
3122         void remove(final DomElement element) {
3123             elements_.remove(element);
3124             sorted_ = elements_.size() < 2;
3125         }
3126     }
3127 }