View Javadoc
1   /*
2    * Copyright (c) 2002-2026 Gargoyle Software Inc.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    * https://www.apache.org/licenses/LICENSE-2.0
8    *
9    * Unless required by applicable law or agreed to in writing, software
10   * distributed under the License is distributed on an "AS IS" BASIS,
11   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12   * See the License for the specific language governing permissions and
13   * limitations under the License.
14   */
15  package org.htmlunit.html;
16  
17  import static org.htmlunit.BrowserVersionFeatures.EVENT_FOCUS_ON_LOAD;
18  import static org.htmlunit.BrowserVersionFeatures.HTTP_HEADER_CH_UA;
19  import static org.htmlunit.html.DomElement.ATTRIBUTE_NOT_DEFINED;
20  
21  import java.io.File;
22  import java.io.IOException;
23  import java.io.ObjectInputStream;
24  import java.io.ObjectOutputStream;
25  import java.io.Serializable;
26  import java.net.MalformedURLException;
27  import java.net.URL;
28  import java.nio.charset.Charset;
29  import java.nio.charset.StandardCharsets;
30  import java.util.ArrayList;
31  import java.util.Arrays;
32  import java.util.Collection;
33  import java.util.Collections;
34  import java.util.Comparator;
35  import java.util.HashMap;
36  import java.util.HashSet;
37  import java.util.Iterator;
38  import java.util.LinkedHashSet;
39  import java.util.List;
40  import java.util.Locale;
41  import java.util.Map;
42  import java.util.Set;
43  import java.util.WeakHashMap;
44  import java.util.concurrent.ConcurrentHashMap;
45  
46  import org.apache.commons.lang3.StringUtils;
47  import org.apache.commons.logging.Log;
48  import org.apache.commons.logging.LogFactory;
49  import org.htmlunit.Cache;
50  import org.htmlunit.ElementNotFoundException;
51  import org.htmlunit.FailingHttpStatusCodeException;
52  import org.htmlunit.History;
53  import org.htmlunit.HttpHeader;
54  import org.htmlunit.OnbeforeunloadHandler;
55  import org.htmlunit.Page;
56  import org.htmlunit.ScriptResult;
57  import org.htmlunit.SgmlPage;
58  import org.htmlunit.TopLevelWindow;
59  import org.htmlunit.WebAssert;
60  import org.htmlunit.WebClient;
61  import org.htmlunit.WebClientOptions;
62  import org.htmlunit.WebRequest;
63  import org.htmlunit.WebResponse;
64  import org.htmlunit.WebWindow;
65  import org.htmlunit.corejs.javascript.Function;
66  import org.htmlunit.corejs.javascript.Script;
67  import org.htmlunit.corejs.javascript.Scriptable;
68  import org.htmlunit.corejs.javascript.ScriptableObject;
69  import org.htmlunit.corejs.javascript.VarScope;
70  import org.htmlunit.css.ComputedCssStyleDeclaration;
71  import org.htmlunit.css.CssStyleSheet;
72  import org.htmlunit.html.impl.SimpleRange;
73  import org.htmlunit.html.parser.HTMLParserDOMBuilder;
74  import org.htmlunit.http.HttpStatus;
75  import org.htmlunit.javascript.AbstractJavaScriptEngine;
76  import org.htmlunit.javascript.HtmlUnitScriptable;
77  import org.htmlunit.javascript.JavaScriptEngine;
78  import org.htmlunit.javascript.PostponedAction;
79  import org.htmlunit.javascript.host.Window;
80  import org.htmlunit.javascript.host.event.BeforeUnloadEvent;
81  import org.htmlunit.javascript.host.event.Event;
82  import org.htmlunit.javascript.host.event.EventTarget;
83  import org.htmlunit.javascript.host.html.HTMLDocument;
84  import org.htmlunit.protocol.javascript.JavaScriptURLConnection;
85  import org.htmlunit.util.MimeType;
86  import org.htmlunit.util.SerializableLock;
87  import org.htmlunit.util.UrlUtils;
88  import org.w3c.dom.Attr;
89  import org.w3c.dom.Comment;
90  import org.w3c.dom.DOMConfiguration;
91  import org.w3c.dom.DOMException;
92  import org.w3c.dom.DOMImplementation;
93  import org.w3c.dom.Document;
94  import org.w3c.dom.DocumentType;
95  import org.w3c.dom.Element;
96  import org.w3c.dom.EntityReference;
97  import org.w3c.dom.ProcessingInstruction;
98  
99  /**
100  * A representation of an HTML page returned from a server.
101  * <p>
102  * This class provides different methods to access the page's content like
103  * {@link #getForms()}, {@link #getAnchors()}, {@link #getElementById(String)}, ... as well as the
104  * very powerful inherited methods {@link #getByXPath(String)} and {@link #getFirstByXPath(String)}
105  * for fine grained user specific access to child nodes.
106  * </p>
107  * <p>
108  * Child elements allowing user interaction provide methods for this purpose like {@link HtmlAnchor#click()},
109  * {@link HtmlInput#type(String)}, {@link HtmlOption#setSelected(boolean)}, ...
110  * </p>
111  * <p>
112  * HtmlPage instances should not be instantiated directly. They will be returned by {@link WebClient#getPage(String)}
113  * when the content type of the server's response is <code>text/html</code> (or one of its variations).<br>
114  * <br>
115  * <b>Example:</b><br>
116  * <br>
117  * <code>
118  * final HtmlPage page = webClient.{@link WebClient#getPage(String) getPage}("http://mywebsite/some/page.html");
119  * </code>
120  * </p>
121  *
122  * @author Mike Bowler
123  * @author Alex Nikiforoff
124  * @author Noboru Sinohara
125  * @author David K. Taylor
126  * @author Andreas Hangler
127  * @author Christian Sell
128  * @author Chris Erskine
129  * @author Marc Guillemot
130  * @author Ahmed Ashour
131  * @author Daniel Gredler
132  * @author Dmitri Zoubkov
133  * @author Sudhan Moghe
134  * @author Ethan Glasser-Camp
135  * @author Tom Anderson
136  * @author Ronald Brill
137  * @author Frank Danek
138  * @author Joerg Werner
139  * @author Atsushi Nakagawa
140  * @author Rural Hunter
141  * @author Ronny Shapiro
142  * @author Lai Quang Duong
143  * @author Sven Strickroth
144  */
145 @SuppressWarnings("PMD.TooManyFields")
146 public class HtmlPage extends SgmlPage {
147 
148     private static final Log LOG = LogFactory.getLog(HtmlPage.class);
149 
150     private static final Comparator<DomElement> DOCUMENT_POSITION_COMPERATOR = new DocumentPositionComparator();
151 
152     private HTMLParserDOMBuilder domBuilder_;
153     private transient Charset originalCharset_;
154     private final Object lock_ = new SerializableLock(); // used for synchronization
155 
156     private Map<String, MappedElementIndexEntry> idMap_ = new ConcurrentHashMap<>();
157     private Map<String, MappedElementIndexEntry> nameMap_ = new ConcurrentHashMap<>();
158     // The id/name lookup index is built lazily on first use. Until then,
159     // notifyNodeAdded / fireAttributeChange skip the per-element index updates.
160     // Reads must call ensureMappedElementsBuilt() before consulting idMap_/nameMap_.
161     private boolean mappedElementsBuilt_;
162 
163     private List<BaseFrameElement> frameElements_ = new ArrayList<>();
164     private int parserCount_;
165     private int snippetParserCount_;
166     private int inlineSnippetParserCount_;
167     private Collection<HtmlAttributeChangeListener> attributeListeners_;
168     private List<PostponedAction> afterLoadActions_ = Collections.synchronizedList(new ArrayList<>());
169     private boolean cleaning_;
170     private HtmlBase base_;
171     private URL baseUrl_;
172     private List<AutoCloseable> autoCloseableList_;
173     private ElementFromPointHandler elementFromPointHandler_;
174     private DomElement elementWithFocus_;
175     private List<SimpleRange> selectionRanges_ = new ArrayList<>(3);
176 
177     private transient ComputedStylesCache computedStylesCache_;
178 
179     private static final HashSet<String> TABBABLE_TAGS =
180             new HashSet<>(Arrays.asList(HtmlAnchor.TAG_NAME, HtmlArea.TAG_NAME,
181                     HtmlButton.TAG_NAME, HtmlInput.TAG_NAME, HtmlObject.TAG_NAME,
182                     HtmlSelect.TAG_NAME, HtmlTextArea.TAG_NAME));
183     private static final HashSet<String> ACCEPTABLE_TAG_NAMES =
184             new HashSet<>(Arrays.asList(HtmlAnchor.TAG_NAME, HtmlArea.TAG_NAME,
185                     HtmlButton.TAG_NAME, HtmlInput.TAG_NAME, HtmlLabel.TAG_NAME,
186                     HtmlLegend.TAG_NAME, HtmlTextArea.TAG_NAME));
187 
188     /** Definition of special cases for the smart DomHtmlAttributeChangeListenerImpl. */
189     private static final Set<String> ATTRIBUTES_AFFECTING_PARENT = new HashSet<>(Arrays.asList(
190             "style",
191             "class",
192             "height",
193             "width"));
194 
195     static class DocumentPositionComparator implements Comparator<DomElement>, Serializable {
196         @Override
197         public int compare(final DomElement elt1, final DomElement elt2) {
198             final short relation = elt1.compareDocumentPosition(elt2);
199             if (relation == 0) {
200                 return 0; // same node
201             }
202             if ((relation & DOCUMENT_POSITION_CONTAINS) != 0 || (relation & DOCUMENT_POSITION_PRECEDING) != 0) {
203                 return 1;
204             }
205 
206             return -1;
207         }
208     }
209 
210     /**
211      * Creates an instance of HtmlPage.
212      * An HtmlPage instance is normally retrieved with {@link WebClient#getPage(String)}.
213      *
214      * @param webResponse the web response that was used to create this page
215      * @param webWindow the window that this page is being loaded into
216      */
217     public HtmlPage(final WebResponse webResponse, final WebWindow webWindow) {
218         super(webResponse, webWindow);
219     }
220 
221     /**
222      * {@inheritDoc}
223      */
224     @Override
225     public HtmlPage getPage() {
226         return this;
227     }
228 
229     /**
230      * {@inheritDoc}
231      */
232     @Override
233     public boolean hasCaseSensitiveTagNames() {
234         return false;
235     }
236 
237     /**
238      * Initialize this page.
239      * @throws IOException if an IO problem occurs
240      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
241      *         {@link org.htmlunit.WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set
242      *         to true.
243      */
244     @Override
245     public void initialize() throws IOException, FailingHttpStatusCodeException {
246         final WebWindow enclosingWindow = getEnclosingWindow();
247         final boolean isAboutBlank = getUrl() == UrlUtils.URL_ABOUT_BLANK;
248         if (isAboutBlank) {
249             // a frame contains first a faked "about:blank" before its real content specified by src gets loaded
250             if (enclosingWindow instanceof FrameWindow window
251                     && !window.getFrameElement().isContentLoaded()) {
252                 return;
253             }
254 
255             // save the URL that should be used to resolve relative URLs in this page
256             if (enclosingWindow instanceof TopLevelWindow topWindow) {
257                 final WebWindow openerWindow = topWindow.getOpener();
258                 if (openerWindow != null && openerWindow.getEnclosedPage() != null) {
259                     baseUrl_ = openerWindow.getEnclosedPage().getWebResponse().getWebRequest().getUrl();
260                 }
261             }
262         }
263 
264         if (!isAboutBlank) {
265             setReadyState(READY_STATE_INTERACTIVE);
266             getDocumentElement().setReadyState(READY_STATE_INTERACTIVE);
267             executeEventHandlersIfNeeded(Event.TYPE_READY_STATE_CHANGE);
268         }
269 
270         executeDeferredScriptsIfNeeded();
271 
272         executeEventHandlersIfNeeded(Event.TYPE_DOM_DOCUMENT_LOADED);
273 
274         // postponed actions are more or less the async scripts,
275         // they are running in real browsers whenever the download is done
276         processPostponedActionsIfNeeded();
277 
278         loadFrames();
279 
280         // don't set the ready state if we really load the blank page into the window
281         // see Node.initInlineFrameIfNeeded()
282         if (!isAboutBlank) {
283             setReadyState(READY_STATE_COMPLETE);
284             getDocumentElement().setReadyState(READY_STATE_COMPLETE);
285             executeEventHandlersIfNeeded(Event.TYPE_READY_STATE_CHANGE);
286         }
287 
288         // frame initialization has a different order
289         boolean isFrameWindow = enclosingWindow instanceof FrameWindow;
290         boolean isFirstPageInFrameWindow = false;
291         if (isFrameWindow) {
292             isFrameWindow = ((FrameWindow) enclosingWindow).getFrameElement() instanceof HtmlFrame;
293 
294             final History hist = enclosingWindow.getHistory();
295             if (hist.getLength() > 0 && UrlUtils.URL_ABOUT_BLANK == hist.getUrl(0)) {
296                 isFirstPageInFrameWindow = hist.getLength() <= 2;
297             }
298             else {
299                 isFirstPageInFrameWindow = enclosingWindow.getHistory().getLength() < 2;
300             }
301         }
302 
303         if (isFrameWindow && !isFirstPageInFrameWindow) {
304             executeEventHandlersIfNeeded(Event.TYPE_LOAD);
305         }
306 
307         for (final BaseFrameElement frameElement : new ArrayList<>(frameElements_)) {
308             if (frameElement instanceof HtmlFrame) {
309                 final Page page = frameElement.getEnclosedWindow().getEnclosedPage();
310                 if (page != null && page.isHtmlPage()) {
311                     ((HtmlPage) page).executeEventHandlersIfNeeded(Event.TYPE_LOAD);
312                 }
313             }
314         }
315 
316         if (!isFrameWindow) {
317             executeEventHandlersIfNeeded(Event.TYPE_LOAD);
318 
319             if (!isAboutBlank && enclosingWindow.getWebClient().isJavaScriptEnabled()
320                     && hasFeature(EVENT_FOCUS_ON_LOAD)) {
321                 final HtmlElement body = getBody();
322                 if (body != null) {
323                     final Event event = new Event((Window) enclosingWindow.getScriptableObject(), Event.TYPE_FOCUS);
324                     body.fireEvent(event);
325                 }
326             }
327         }
328 
329         try {
330             while (!afterLoadActions_.isEmpty()) {
331                 final PostponedAction action = afterLoadActions_.remove(0);
332                 action.execute();
333             }
334         }
335         catch (final IOException e) {
336             throw e;
337         }
338         catch (final Exception e) {
339             throw new RuntimeException(e);
340         }
341         executeRefreshIfNeeded();
342     }
343 
344     /**
345      * Adds an action that should be executed once the page has been loaded.
346      * @param action the action
347      */
348     void addAfterLoadAction(final PostponedAction action) {
349         afterLoadActions_.add(action);
350     }
351 
352     /**
353      * Clean up this page.
354      */
355     @Override
356     public void cleanUp() {
357         //To avoid endless recursion caused by window.close() in onUnload
358         if (cleaning_) {
359             return;
360         }
361 
362         cleaning_ = true;
363         try {
364             super.cleanUp();
365             executeEventHandlersIfNeeded(Event.TYPE_UNLOAD);
366             deregisterFramesIfNeeded();
367         }
368         finally {
369             cleaning_ = false;
370 
371             if (autoCloseableList_ != null) {
372                 for (final AutoCloseable closeable : new ArrayList<>(autoCloseableList_)) {
373                     try {
374                         closeable.close();
375                     }
376                     catch (final Exception e) {
377                         LOG.error("Closing the autoclosable " + closeable + " failed", e);
378                     }
379                 }
380             }
381         }
382     }
383 
384     /**
385      * {@inheritDoc}
386      */
387     @Override
388     public HtmlElement getDocumentElement() {
389         return (HtmlElement) super.getDocumentElement();
390     }
391 
392     /**
393      * Returns the document's {@code body} element.
394      *
395      * @return the document's {@code body} element, or {@code null} if it does
396      *         not exist
397      */
398     public HtmlBody getBody() {
399         final DomElement doc = getDocumentElement();
400         if (doc != null) {
401             for (final DomNode node : doc.getChildren()) {
402                 if (node instanceof HtmlBody body) {
403                     return body;
404                 }
405             }
406         }
407         return null;
408     }
409 
410     /**
411      * Returns the head element.
412      * @return the head element
413      */
414     public HtmlElement getHead() {
415         final DomElement doc = getDocumentElement();
416         if (doc != null) {
417             for (final DomNode node : doc.getChildren()) {
418                 if (node instanceof HtmlHead) {
419                     return (HtmlElement) node;
420                 }
421             }
422         }
423         return null;
424     }
425 
426     /**
427      * {@inheritDoc}
428      */
429     @Override
430     public Document getOwnerDocument() {
431         return null;
432     }
433 
434     /**
435      * {@inheritDoc}
436      * Not yet implemented.
437      */
438     @Override
439     public org.w3c.dom.Node importNode(final org.w3c.dom.Node importedNode, final boolean deep) {
440         throw new UnsupportedOperationException("HtmlPage.importNode is not yet implemented.");
441     }
442 
443     /**
444      * {@inheritDoc}
445      * Not yet implemented.
446      */
447     @Override
448     public String getInputEncoding() {
449         throw new UnsupportedOperationException("HtmlPage.getInputEncoding is not yet implemented.");
450     }
451 
452     /**
453      * {@inheritDoc}
454      */
455     @Override
456     public String getXmlEncoding() {
457         return null;
458     }
459 
460     /**
461      * {@inheritDoc}
462      */
463     @Override
464     public boolean getXmlStandalone() {
465         return false;
466     }
467 
468     /**
469      * {@inheritDoc}
470      * Not yet implemented.
471      */
472     @Override
473     public void setXmlStandalone(final boolean xmlStandalone) throws DOMException {
474         throw new UnsupportedOperationException("HtmlPage.setXmlStandalone is not yet implemented.");
475     }
476 
477     /**
478      * {@inheritDoc}
479      */
480     @Override
481     public String getXmlVersion() {
482         return null;
483     }
484 
485     /**
486      * {@inheritDoc}
487      * Not yet implemented.
488      */
489     @Override
490     public void setXmlVersion(final String xmlVersion) throws DOMException {
491         throw new UnsupportedOperationException("HtmlPage.setXmlVersion is not yet implemented.");
492     }
493 
494     /**
495      * {@inheritDoc}
496      * Not yet implemented.
497      */
498     @Override
499     public boolean getStrictErrorChecking() {
500         throw new UnsupportedOperationException("HtmlPage.getStrictErrorChecking is not yet implemented.");
501     }
502 
503     /**
504      * {@inheritDoc}
505      * Not yet implemented.
506      */
507     @Override
508     public void setStrictErrorChecking(final boolean strictErrorChecking) {
509         throw new UnsupportedOperationException("HtmlPage.setStrictErrorChecking is not yet implemented.");
510     }
511 
512     /**
513      * {@inheritDoc}
514      * Not yet implemented.
515      */
516     @Override
517     public String getDocumentURI() {
518         throw new UnsupportedOperationException("HtmlPage.getDocumentURI is not yet implemented.");
519     }
520 
521     /**
522      * {@inheritDoc}
523      * Not yet implemented.
524      */
525     @Override
526     public void setDocumentURI(final String documentURI) {
527         throw new UnsupportedOperationException("HtmlPage.setDocumentURI is not yet implemented.");
528     }
529 
530     /**
531      * {@inheritDoc}
532      * Not yet implemented.
533      */
534     @Override
535     public org.w3c.dom.Node adoptNode(final org.w3c.dom.Node source) throws DOMException {
536         throw new UnsupportedOperationException("HtmlPage.adoptNode is not yet implemented.");
537     }
538 
539     /**
540      * {@inheritDoc}
541      * Not yet implemented.
542      */
543     @Override
544     public DOMConfiguration getDomConfig() {
545         throw new UnsupportedOperationException("HtmlPage.getDomConfig is not yet implemented.");
546     }
547 
548     /**
549      * {@inheritDoc}
550      * Not yet implemented.
551      */
552     @Override
553     public org.w3c.dom.Node renameNode(final org.w3c.dom.Node newNode, final String namespaceURI,
554         final String qualifiedName) throws DOMException {
555         throw new UnsupportedOperationException("HtmlPage.renameNode is not yet implemented.");
556     }
557 
558     /**
559      * {@inheritDoc}
560      */
561     @Override
562     public Charset getCharset() {
563         if (originalCharset_ == null) {
564             originalCharset_ = getWebResponse().getContentCharset();
565         }
566         return originalCharset_;
567     }
568 
569     /**
570      * {@inheritDoc}
571      */
572     @Override
573     public String getContentType() {
574         return getWebResponse().getContentType();
575     }
576 
577     /**
578      * {@inheritDoc}
579      * Not yet implemented.
580      */
581     @Override
582     public DOMImplementation getImplementation() {
583         throw new UnsupportedOperationException("HtmlPage.getImplementation is not yet implemented.");
584     }
585 
586     /**
587      * {@inheritDoc}
588      * @param tagName the tag name, preferably in lowercase
589      */
590     @Override
591     public DomElement createElement(String tagName) {
592         if (tagName.indexOf(':') == -1) {
593             tagName = org.htmlunit.util.StringUtils.toRootLowerCase(tagName);
594         }
595         return getWebClient().getPageCreator().getHtmlParser().getFactory(tagName)
596                     .createElementNS(this, null, tagName, null);
597     }
598 
599     /**
600      * {@inheritDoc}
601      */
602     @Override
603     public DomElement createElementNS(final String namespaceURI, final String qualifiedName) {
604         return getWebClient().getPageCreator().getHtmlParser()
605                 .getElementFactory(this, namespaceURI, qualifiedName, false, true)
606                 .createElementNS(this, namespaceURI, qualifiedName, null);
607     }
608 
609     /**
610      * {@inheritDoc}
611      * Not yet implemented.
612      */
613     @Override
614     public Attr createAttributeNS(final String namespaceURI, final String qualifiedName) {
615         throw new UnsupportedOperationException("HtmlPage.createAttributeNS is not yet implemented.");
616     }
617 
618     /**
619      * {@inheritDoc}
620      * Not yet implemented.
621      */
622     @Override
623     public EntityReference createEntityReference(final String id) {
624         throw new UnsupportedOperationException("HtmlPage.createEntityReference is not yet implemented.");
625     }
626 
627     /**
628      * {@inheritDoc}
629      * Not yet implemented.
630      */
631     @Override
632     public ProcessingInstruction createProcessingInstruction(final String namespaceURI, final String qualifiedName) {
633         throw new UnsupportedOperationException("HtmlPage.createProcessingInstruction is not yet implemented.");
634     }
635 
636     /**
637      * {@inheritDoc}
638      */
639     @Override
640     public DomElement getElementById(final String elementId) {
641         if (elementId != null) {
642             ensureMappedElementsBuilt();
643             final MappedElementIndexEntry elements = idMap_.get(elementId);
644             if (elements != null) {
645                 return elements.first();
646             }
647         }
648         return null;
649     }
650 
651     /**
652      * Returns the {@link HtmlAnchor} with the specified name.
653      *
654      * @param name the name to search by
655      * @return the {@link HtmlAnchor} with the specified name
656      * @throws ElementNotFoundException if the anchor could not be found
657      */
658     public HtmlAnchor getAnchorByName(final String name) throws ElementNotFoundException {
659         return getDocumentElement().getOneHtmlElementByAttribute("a", DomElement.NAME_ATTRIBUTE, name);
660     }
661 
662     /**
663      * Returns the {@link HtmlAnchor} with the specified href.
664      *
665      * @param href the string to search by
666      * @return the HtmlAnchor
667      * @throws ElementNotFoundException if the anchor could not be found
668      */
669     public HtmlAnchor getAnchorByHref(final String href) throws ElementNotFoundException {
670         return getDocumentElement().getOneHtmlElementByAttribute("a", "href", href);
671     }
672 
673     /**
674      * Returns a list of all anchors contained in this page.
675      * @return the list of {@link HtmlAnchor} in this page
676      */
677     public List<HtmlAnchor> getAnchors() {
678         return getDocumentElement().getElementsByTagNameImpl("a");
679     }
680 
681     /**
682      * Returns the first anchor with the specified text.
683      * @param text the text to search for
684      * @return the first anchor that was found
685      * @throws ElementNotFoundException if no anchors are found with the specified text
686      */
687     public HtmlAnchor getAnchorByText(final String text) throws ElementNotFoundException {
688         WebAssert.notNull("text", text);
689 
690         for (final HtmlAnchor anchor : getAnchors()) {
691             if (text.equals(anchor.asNormalizedText())) {
692                 return anchor;
693             }
694         }
695         throw new ElementNotFoundException("a", "<text>", text);
696     }
697 
698     /**
699      * Returns the first form that matches the specified name.
700      * @param name the name to search for
701      * @return the first form
702      * @throws ElementNotFoundException If no forms match the specified result.
703      */
704     public HtmlForm getFormByName(final String name) throws ElementNotFoundException {
705         final List<HtmlForm> forms = getDocumentElement()
706                 .getElementsByAttribute("form", DomElement.NAME_ATTRIBUTE, name);
707         if (forms.isEmpty()) {
708             throw new ElementNotFoundException("form", DomElement.NAME_ATTRIBUTE, name);
709         }
710         return forms.get(0);
711     }
712 
713     /**
714      * Returns a list of all the forms in this page.
715      * @return all the forms in this page
716      */
717     public List<HtmlForm> getForms() {
718         return getDocumentElement().getElementsByTagNameImpl("form");
719     }
720 
721     /**
722      * Given a relative URL (ie <code>/foo</code>), returns a fully-qualified URL based on
723      * the URL that was used to load this page.
724      *
725      * @param relativeUrl the relative URL
726      * @return the fully-qualified URL for the specified relative URL
727      * @throws MalformedURLException if an error occurred when creating a URL object
728      */
729     public URL getFullyQualifiedUrl(String relativeUrl) throws MalformedURLException {
730         // to handle http: and http:/ in FF (Bug #474)
731         boolean incorrectnessNotified = false;
732         while (relativeUrl.startsWith("http:") && !relativeUrl.startsWith("http://")) {
733             if (!incorrectnessNotified) {
734                 notifyIncorrectness("Incorrect URL \"" + relativeUrl + "\" has been corrected");
735                 incorrectnessNotified = true;
736             }
737             relativeUrl = "http:/" + relativeUrl.substring(5);
738         }
739 
740         return WebClient.expandUrl(getBaseURL(), relativeUrl);
741     }
742 
743     /**
744      * Given a target attribute value, resolve the target using a base target for the page.
745      *
746      * @param elementTarget the target specified as an attribute of the element
747      * @return the resolved target to use for the element
748      */
749     public String getResolvedTarget(final String elementTarget) {
750         final String resolvedTarget;
751         if (base_ == null) {
752             resolvedTarget = elementTarget;
753         }
754         else if (elementTarget != null && !elementTarget.isEmpty()) {
755             resolvedTarget = elementTarget;
756         }
757         else {
758             resolvedTarget = base_.getTargetAttribute();
759         }
760         return resolvedTarget;
761     }
762 
763     /**
764      * Returns a list of ids (strings) that correspond to the tabbable elements
765      * in this page. Return them in the same order specified in {@link #getTabbableElements}
766      *
767      * @return the list of id's
768      */
769     public List<String> getTabbableElementIds() {
770         final List<String> list = new ArrayList<>();
771 
772         for (final HtmlElement element : getTabbableElements()) {
773             list.add(element.getId());
774         }
775 
776         return Collections.unmodifiableList(list);
777     }
778 
779     /**
780      * Returns a list of all elements that are tabbable in the order that will
781      * be used for tabbing.
782      * <p>
783      * The rules for determining tab order are as follows:
784      * </p>
785      * <ol>
786      *   <li>Those elements that support the tabindex attribute and assign a
787      *   positive value to it are navigated first. Navigation proceeds from the
788      *   element with the lowest tabindex value to the element with the highest
789      *   value. Values need not be sequential nor must they begin with any
790      *   particular value. Elements that have identical tabindex values should
791      *   be navigated in the order they appear in the character stream.</li>
792      *   <li>Those elements that do not support the tabindex attribute or
793      *   support it and assign it a value of "0" are navigated next. These
794      *   elements are navigated in the order they appear in the character
795      *   stream.</li>
796      *   <li>Elements that are disabled do not participate in the tabbing
797      *   order.</li>
798      * </ol>
799      * <p>
800      * Additionally, the value of tabindex must be within 0 and 32767. Any
801      * values outside this range will be ignored.
802      * </p>
803      * <p>
804      * The following elements support the <code>tabindex</code> attribute:
805      * A, AREA, BUTTON, INPUT, OBJECT, SELECT, and TEXTAREA.
806      * </p>
807      *
808      * @return all the tabbable elements in proper tab order
809      */
810     public List<HtmlElement> getTabbableElements() {
811         final List<HtmlElement> tabbableElements = new ArrayList<>();
812         for (final HtmlElement element : getHtmlElementDescendants()) {
813             final String tagName = element.getTagName();
814             if (TABBABLE_TAGS.contains(tagName)) {
815                 final boolean disabled = element.isDisabledElementAndDisabled();
816                 if (!disabled && !HtmlElement.TAB_INDEX_OUT_OF_BOUNDS.equals(element.getTabIndex())) {
817                     tabbableElements.add(element);
818                 }
819             }
820         }
821         tabbableElements.sort(createTabOrderComparator());
822         return Collections.unmodifiableList(tabbableElements);
823     }
824 
825     private static Comparator<HtmlElement> createTabOrderComparator() {
826         return (element1, element2) -> {
827             final Short i1 = element1.getTabIndex();
828             final Short i2 = element2.getTabIndex();
829 
830             final short index1;
831             if (i1 == null) {
832                 index1 = -1;
833             }
834             else {
835                 index1 = i1.shortValue();
836             }
837 
838             final short index2;
839             if (i2 == null) {
840                 index2 = -1;
841             }
842             else {
843                 index2 = i2.shortValue();
844             }
845 
846             final int result;
847             if (index1 > 0 && index2 > 0) {
848                 result = index1 - index2;
849             }
850             else if (index1 > 0) {
851                 result = -1;
852             }
853             else if (index2 > 0) {
854                 result = 1;
855             }
856             else if (index1 == index2) {
857                 result = 0;
858             }
859             else {
860                 result = index2 - index1;
861             }
862 
863             return result;
864         };
865     }
866 
867     /**
868      * Returns the HTML element that is assigned to the specified access key. An
869      * access key (aka mnemonic key) is used for keyboard navigation of the
870      * page.
871      * <p>
872      * Only the following HTML elements may have <code>accesskey</code>s defined: A, AREA,
873      * BUTTON, INPUT, LABEL, LEGEND, and TEXTAREA.
874      * </p>
875      *
876      * @param accessKey the key to look for
877      * @return the HTML element that is assigned to the specified key or null
878      *      if no elements can be found that match the specified key.
879      */
880     public HtmlElement getHtmlElementByAccessKey(final char accessKey) {
881         final List<HtmlElement> elements = getHtmlElementsByAccessKey(accessKey);
882         if (elements.isEmpty()) {
883             return null;
884         }
885         return elements.get(0);
886     }
887 
888     /**
889      * Returns all the HTML elements that are assigned to the specified access key. An
890      * access key (aka mnemonic key) is used for keyboard navigation of the
891      * page.
892      * <p>
893      * The HTML specification seems to indicate that one accesskey cannot be used
894      * for multiple elements however Internet Explorer does seem to support this.
895      * It's worth noting that Firefox does not support multiple elements with one
896      * access key so you are making your HTML browser specific if you rely on this
897      * feature.
898      * </p>
899      *
900      * <p>
901      * Only the following HTML elements may have <code>accesskey</code>s defined: A, AREA,
902      * BUTTON, INPUT, LABEL, LEGEND, and TEXTAREA.
903      * </p>
904      *
905      * @param accessKey the key to look for
906      * @return the elements that are assigned to the specified accesskey
907      */
908     public List<HtmlElement> getHtmlElementsByAccessKey(final char accessKey) {
909         final List<HtmlElement> elements = new ArrayList<>();
910 
911         final String searchString = Character.toString(accessKey).toLowerCase(Locale.ROOT);
912         for (final HtmlElement element : getHtmlElementDescendants()) {
913             if (ACCEPTABLE_TAG_NAMES.contains(element.getTagName())) {
914                 final String accessKeyAttribute = element.getAttributeDirect("accesskey");
915                 if (searchString.equalsIgnoreCase(accessKeyAttribute)) {
916                     elements.add(element);
917                 }
918             }
919         }
920 
921         return elements;
922     }
923 
924     /**
925      * <p>Executes the specified JavaScript code within the page. The usage would be similar to what can
926      * be achieved to execute JavaScript in the current page by entering "javascript:...some JS code..."
927      * in the URL field of a native browser.</p>
928      * <p><b>Note:</b> the provided code won't be executed if JavaScript has been disabled on the WebClient
929      * (see {@link org.htmlunit.WebClient#isJavaScriptEnabled()}).</p>
930      * @param sourceCode the JavaScript code to execute
931      * @return a ScriptResult which will contain both the current page (which may be different from
932      *         the previous page) and a JavaScript result object
933      */
934     public ScriptResult executeJavaScript(final String sourceCode) {
935         return executeJavaScript(sourceCode, "injected script", 1);
936     }
937 
938     /**
939      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
940      * <p>
941      * Execute the specified JavaScript if a JavaScript engine was successfully
942      * instantiated. If this JavaScript causes the current page to be reloaded
943      * (through location="" or form.submit()) then return the new page, otherwise
944      * return the current page.
945      * </p>
946      * <p><b>Please note:</b> Although this method is public, it is not intended for
947      * general execution of JavaScript. Users of HtmlUnit should interact with the pages
948      * as a user would by clicking on buttons or links and having the JavaScript event
949      * handlers execute as needed.
950      * </p>
951      *
952      * @param sourceCode the JavaScript code to execute
953      * @param sourceName the name for this chunk of code (will be displayed in error messages)
954      * @param startLine the line at which the script source starts
955      * @return a ScriptResult which will contain both the current page (which may be different from
956      *         the previous page) and a JavaScript result object.
957      */
958     public ScriptResult executeJavaScript(String sourceCode, final String sourceName, final int startLine) {
959         if (!getWebClient().isJavaScriptEnabled()) {
960             return new ScriptResult(JavaScriptEngine.UNDEFINED);
961         }
962 
963         if (org.htmlunit.util.StringUtils.startsWithIgnoreCase(sourceCode,
964                                                 JavaScriptURLConnection.JAVASCRIPT_PREFIX)) {
965             sourceCode = sourceCode.substring(JavaScriptURLConnection.JAVASCRIPT_PREFIX.length()).trim();
966             if (sourceCode.startsWith("return ")) {
967                 sourceCode = sourceCode.substring("return ".length());
968             }
969         }
970 
971         final Window window = getEnclosingWindow().getScriptableObject();
972         final VarScope scope = ScriptableObject.getTopLevelScope(window.getParentScope());
973 
974         final Object result = getWebClient().getJavaScriptEngine()
975                 .execute(this, scope, sourceCode, sourceName, startLine);
976         return new ScriptResult(result);
977     }
978 
979     /** Various possible external JavaScript file loading results. */
980     enum JavaScriptLoadResult {
981         /** The load was aborted and nothing was done. */
982         NOOP,
983         /** The load was aborted and nothing was done. */
984         NO_CONTENT,
985         /** The external JavaScript file was downloaded and compiled successfully. */
986         SUCCESS,
987         /** The external JavaScript file was not downloaded successfully. */
988         DOWNLOAD_ERROR,
989         /** The external JavaScript file was downloaded but was not compiled successfully. */
990         COMPILATION_ERROR
991     }
992 
993     /**
994      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
995      *
996      * @param srcAttribute the source attribute from the script tag
997      * @param scriptCharset the charset from the script tag
998      * @return the result of loading the specified external JavaScript file
999      * @throws FailingHttpStatusCodeException if the request's status code indicates a request
1000      *         failure and the {@link WebClient} was configured to throw exceptions on failing
1001      *         HTTP status codes
1002      */
1003     JavaScriptLoadResult loadExternalJavaScriptFile(final String srcAttribute,
1004                             final Charset scriptCharset, final boolean crossorigin)
1005         throws FailingHttpStatusCodeException {
1006 
1007         final WebClient client = getWebClient();
1008         if (org.htmlunit.util.StringUtils.isBlank(srcAttribute) || !client.isJavaScriptEnabled()) {
1009             return JavaScriptLoadResult.NOOP;
1010         }
1011 
1012         final URL scriptURL;
1013         try {
1014             scriptURL = getFullyQualifiedUrl(srcAttribute);
1015             final String protocol = scriptURL.getProtocol();
1016             if ("javascript".equals(protocol)) {
1017                 if (LOG.isInfoEnabled()) {
1018                     LOG.info("Ignoring script src [" + srcAttribute + "]");
1019                 }
1020                 return JavaScriptLoadResult.NOOP;
1021             }
1022             if (!"http".equals(protocol) && !"https".equals(protocol)
1023                     && !"data".equals(protocol) && !"file".equals(protocol)) {
1024                 client.getJavaScriptErrorListener().malformedScriptURL(this, srcAttribute,
1025                         new MalformedURLException("unknown protocol: '" + protocol + "'"));
1026                 return JavaScriptLoadResult.NOOP;
1027             }
1028         }
1029         catch (final MalformedURLException e) {
1030             client.getJavaScriptErrorListener().malformedScriptURL(this, srcAttribute, e);
1031             return JavaScriptLoadResult.NOOP;
1032         }
1033 
1034         final Object script;
1035         try {
1036             script = loadJavaScriptFromUrl(scriptURL, scriptCharset, crossorigin);
1037         }
1038         catch (final IOException e) {
1039             client.getJavaScriptErrorListener().loadScriptError(this, scriptURL, e);
1040             return JavaScriptLoadResult.DOWNLOAD_ERROR;
1041         }
1042         catch (final FailingHttpStatusCodeException e) {
1043             if (e.getStatusCode() == HttpStatus.NO_CONTENT_204) {
1044                 return JavaScriptLoadResult.NO_CONTENT;
1045             }
1046             client.getJavaScriptErrorListener().loadScriptError(this, scriptURL, e);
1047             throw e;
1048         }
1049 
1050         if (script == null) {
1051             return JavaScriptLoadResult.COMPILATION_ERROR;
1052         }
1053 
1054         final Window window = getEnclosingWindow().getScriptableObject();
1055         final VarScope scope = ScriptableObject.getTopLevelScope(window.getParentScope());
1056 
1057         @SuppressWarnings("unchecked")
1058         final AbstractJavaScriptEngine<Object> engine = (AbstractJavaScriptEngine<Object>) client.getJavaScriptEngine();
1059         engine.execute(this, scope, script);
1060         return JavaScriptLoadResult.SUCCESS;
1061     }
1062 
1063     /**
1064      * Loads JavaScript from the specified URL. This method may return {@code null} if
1065      * there is a problem loading the code from the specified URL.
1066      *
1067      * @param url the URL of the script
1068      * @param scriptCharset the charset from the script tag
1069      * @return the content of the file, or {@code null} if we ran into a compile error
1070      * @throws IOException if there is a problem downloading the JavaScript file
1071      * @throws FailingHttpStatusCodeException if the request's status code indicates a request
1072      *         failure and the {@link WebClient} was configured to throw exceptions on failing
1073      *         HTTP status codes
1074      */
1075     private Object loadJavaScriptFromUrl(final URL url, final Charset scriptCharset,
1076                     final boolean crossorigin) throws IOException,
1077         FailingHttpStatusCodeException {
1078 
1079         final WebRequest referringRequest = getWebResponse().getWebRequest();
1080 
1081         final WebClient client = getWebClient();
1082         final WebRequest request = new WebRequest(url);
1083         // copy all headers from the referring request
1084         request.setAdditionalHeaders(new HashMap<>(referringRequest.getAdditionalHeaders()));
1085 
1086         // at least overwrite this headers
1087         request.setAdditionalHeader(HttpHeader.ACCEPT, client.getBrowserVersion().getScriptAcceptHeader());
1088 
1089         request.setFetchDestination(WebRequest.FetchDestination.SCRIPT);
1090         request.setRequestingUrl(referringRequest.getUrl());
1091         request.setFetchModeOverride(WebRequest.FetchMode.NO_CORS);
1092 
1093         request.setRefererHeader(referringRequest.getUrl());
1094         request.setCharset(scriptCharset);
1095 
1096         // use info from script tag or fall back to utf-8
1097         // https://www.rfc-editor.org/rfc/rfc9239#section-4.2
1098         if (scriptCharset != null) {
1099             request.setDefaultResponseContentCharset(scriptCharset);
1100         }
1101         else {
1102             request.setDefaultResponseContentCharset(StandardCharsets.UTF_8);
1103         }
1104 
1105         if (crossorigin) {
1106             request.setFetchModeOverride(WebRequest.FetchMode.CORS);
1107 
1108             if (client.getBrowserVersion().hasFeature(HTTP_HEADER_CH_UA)) {
1109                 request.setAdditionalHeader(HttpHeader.ORIGIN,
1110                         UrlUtils.getUrlWithProtocolAndAuthority(url).toExternalForm());
1111             }
1112         }
1113 
1114         // our cache is a bit strange;
1115         // loadWebResponse check the cache for the web response
1116         // AND also fixes the request url for the following cache lookups
1117         final WebResponse response = client.loadWebResponse(request);
1118 
1119         // now we can look into the cache with the fixed request for
1120         // a cached script
1121         final Cache cache = client.getCache();
1122         final Object cachedScript = cache.getCachedObject(request);
1123         if (cachedScript instanceof Script) {
1124             return cachedScript;
1125         }
1126 
1127         client.printContentIfNecessary(response);
1128         client.throwFailingHttpStatusCodeExceptionIfNecessary(response);
1129 
1130         final int statusCode = response.getStatusCode();
1131         if (statusCode == HttpStatus.NO_CONTENT_204) {
1132             throw new FailingHttpStatusCodeException(response);
1133         }
1134 
1135         if (!response.isSuccess()) {
1136             throw new IOException("Unable to download JavaScript from '" + url + "' (status " + statusCode + ").");
1137         }
1138 
1139         final String contentType = response.getContentType();
1140         if (contentType != null) {
1141             if (MimeType.isObsoleteJavascriptMimeType(contentType)) {
1142                 getWebClient().getIncorrectnessListener().notify(
1143                         "Obsolete content type encountered: '" + contentType + "' "
1144                                 + "for remotely loaded JavaScript element at '" + url + "'.", this);
1145             }
1146             else if (!MimeType.isJavascriptMimeType(contentType)) {
1147                 getWebClient().getIncorrectnessListener().notify(
1148                         "Expect content type of '" + MimeType.TEXT_JAVASCRIPT + "' "
1149                                 + "for remotely loaded JavaScript element at '" + url + "', "
1150                                 + "but got '" + contentType + "'.", this);
1151             }
1152         }
1153 
1154         final Charset scriptEncoding = response.getContentCharset();
1155         final String scriptCode = response.getContentAsString(scriptEncoding);
1156         if (null != scriptCode) {
1157             final AbstractJavaScriptEngine<?> javaScriptEngine = client.getJavaScriptEngine();
1158 
1159             final Window window = getEnclosingWindow().getScriptableObject();
1160             final VarScope scope = ScriptableObject.getTopLevelScope(window.getParentScope());
1161 
1162             final Object script = javaScriptEngine.compile(this, scope, scriptCode, url.toExternalForm(), 1);
1163             if (script != null && cache.cacheIfPossible(request, response, script)) {
1164                 // no cleanup if the response is stored inside the cache
1165                 return script;
1166             }
1167 
1168             response.cleanUp();
1169             return script;
1170         }
1171 
1172         response.cleanUp();
1173         return null;
1174     }
1175 
1176     /**
1177      * Returns the title of this page or an empty string if the title wasn't specified.
1178      *
1179      * @return the title of this page or an empty string if the title wasn't specified
1180      */
1181     public String getTitleText() {
1182         final HtmlTitle titleElement = getTitleElement();
1183         if (titleElement != null) {
1184             return titleElement.asNormalizedText();
1185         }
1186         return "";
1187     }
1188 
1189     /**
1190      * Sets the text for the title of this page. If there is not a title element
1191      * on this page, then one has to be generated.
1192      * @param message the new text
1193      */
1194     public void setTitleText(final String message) {
1195         HtmlTitle titleElement = getTitleElement();
1196         if (titleElement == null) {
1197             LOG.debug("No title element, creating one");
1198             final HtmlHead head = (HtmlHead) getFirstChildElement(getDocumentElement(), HtmlHead.class);
1199             if (head == null) {
1200                 // perhaps should we create head too?
1201                 throw new IllegalStateException("Headelement was not defined for this page");
1202             }
1203             final Map<String, DomAttr> emptyMap = Collections.emptyMap();
1204             titleElement = new HtmlTitle(HtmlTitle.TAG_NAME, this, emptyMap);
1205             if (head.getFirstChild() != null) {
1206                 head.getFirstChild().insertBefore(titleElement);
1207             }
1208             else {
1209                 head.appendChild(titleElement);
1210             }
1211         }
1212 
1213         titleElement.setNodeValue(message);
1214     }
1215 
1216     /**
1217      * Gets the first child of startElement that is an instance of the given class.
1218      * @param startElement the parent element
1219      * @param clazz the class to search for
1220      * @return {@code null} if no child found
1221      */
1222     private static DomElement getFirstChildElement(final DomElement startElement, final Class<?> clazz) {
1223         if (startElement == null) {
1224             return null;
1225         }
1226         for (final DomElement element : startElement.getChildElements()) {
1227             if (clazz.isInstance(element)) {
1228                 return element;
1229             }
1230         }
1231 
1232         return null;
1233     }
1234 
1235     /**
1236      * Gets the first child of startElement or it's children that is an instance of the given class.
1237      * @param startElement the parent element
1238      * @param clazz the class to search for
1239      * @return {@code null} if no child found
1240      */
1241     private DomElement getFirstChildElementRecursive(final DomElement startElement, final Class<?> clazz) {
1242         if (startElement == null) {
1243             return null;
1244         }
1245         for (final DomElement element : startElement.getChildElements()) {
1246             if (clazz.isInstance(element)) {
1247                 return element;
1248             }
1249             final DomElement childFound = getFirstChildElementRecursive(element, clazz);
1250             if (childFound != null) {
1251                 return childFound;
1252             }
1253         }
1254 
1255         return null;
1256     }
1257 
1258     /**
1259      * Gets the title element for this page. Returns null if one is not found.
1260      *
1261      * @return the title element for this page or null if this is not one
1262      */
1263     private HtmlTitle getTitleElement() {
1264         return (HtmlTitle) getFirstChildElementRecursive(getDocumentElement(), HtmlTitle.class);
1265     }
1266 
1267     /**
1268      * Looks for and executes any appropriate event handlers. Looks for body and frame tags.
1269      * @param eventType either {@link Event#TYPE_LOAD}, {@link Event#TYPE_UNLOAD}, or {@link Event#TYPE_BEFORE_UNLOAD}
1270      * @return {@code true} if user accepted <code>onbeforeunload</code> (not relevant to other events)
1271      */
1272     private boolean executeEventHandlersIfNeeded(final String eventType) {
1273         // If JavaScript isn't enabled, there's nothing for us to do.
1274         if (!getWebClient().isJavaScriptEnabled()) {
1275             return true;
1276         }
1277 
1278         // Execute the specified event on the document element.
1279         final WebWindow window = getEnclosingWindow();
1280         if (window.getScriptableObject() instanceof Window) {
1281             final Event event;
1282             if (Event.TYPE_BEFORE_UNLOAD.equals(eventType)) {
1283                 event = new BeforeUnloadEvent(this, eventType);
1284             }
1285             else {
1286                 event = new Event(this, eventType);
1287             }
1288 
1289             // This is the same as DomElement.fireEvent() and was copied
1290             // here so it could be used with HtmlPage.
1291             if (LOG.isDebugEnabled()) {
1292                 LOG.debug("Firing " + event);
1293             }
1294 
1295             final EventTarget jsNode;
1296             if (Event.TYPE_DOM_DOCUMENT_LOADED.equals(eventType)) {
1297                 jsNode = getScriptableObject();
1298             }
1299             else if (Event.TYPE_READY_STATE_CHANGE.equals(eventType)) {
1300                 jsNode = getDocumentElement().getScriptableObject();
1301             }
1302             else {
1303                 // The load/beforeunload/unload events target Document but paths Window only (tested in Chrome/FF)
1304                 jsNode = window.getScriptableObject();
1305             }
1306 
1307             ((JavaScriptEngine) getWebClient().getJavaScriptEngine()).callSecured(cx -> jsNode.fireEvent(event), this);
1308 
1309             if (!isOnbeforeunloadAccepted(this, event)) {
1310                 return false;
1311             }
1312         }
1313 
1314         // If this page was loaded in a frame, execute the version of the event specified on the frame tag.
1315         if (window instanceof FrameWindow fw) {
1316             final BaseFrameElement frame = fw.getFrameElement();
1317 
1318             // if part of a document fragment, then the load event is not triggered
1319             if (Event.TYPE_LOAD.equals(eventType) && frame.getParentNode() instanceof DomDocumentFragment) {
1320                 return true;
1321             }
1322 
1323             if (frame.hasEventHandlers("on" + eventType)) {
1324                 if (LOG.isDebugEnabled()) {
1325                     LOG.debug("Executing on" + eventType + " handler for " + frame);
1326                 }
1327                 if (window.getScriptableObject() instanceof Window) {
1328                     final Event event;
1329                     if (Event.TYPE_BEFORE_UNLOAD.equals(eventType)) {
1330                         event = new BeforeUnloadEvent(frame, eventType);
1331                     }
1332                     else {
1333                         event = new Event(frame, eventType);
1334                     }
1335                     // This fires the "load" event for the <frame> element which, like all non-window
1336                     // load events, propagates up to Document but not Window.  The "load" event for
1337                     // <frameset> on the other hand, like that of <body>, is handled above where it is
1338                     // fired against Document and directed to Window.
1339                     frame.fireEvent(event);
1340 
1341                     if (!isOnbeforeunloadAccepted((HtmlPage) frame.getPage(), event)) {
1342                         return false;
1343                     }
1344                 }
1345             }
1346         }
1347 
1348         return true;
1349     }
1350 
1351     /**
1352      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1353      *
1354      * @return true if the OnbeforeunloadHandler has accepted to change the page
1355      */
1356     public boolean isOnbeforeunloadAccepted() {
1357         return executeEventHandlersIfNeeded(Event.TYPE_BEFORE_UNLOAD);
1358     }
1359 
1360     private boolean isOnbeforeunloadAccepted(final HtmlPage page, final Event event) {
1361         if (event instanceof BeforeUnloadEvent beforeUnloadEvent) {
1362             if (beforeUnloadEvent.isBeforeUnloadMessageSet()) {
1363                 final OnbeforeunloadHandler handler = getWebClient().getOnbeforeunloadHandler();
1364                 if (handler == null) {
1365                     LOG.warn("document.onbeforeunload() returned a string in event.returnValue,"
1366                             + " but no onbeforeunload handler installed.");
1367                 }
1368                 else {
1369                     final String message = JavaScriptEngine.toString(beforeUnloadEvent.getReturnValue());
1370                     return handler.handleEvent(page, message);
1371                 }
1372             }
1373         }
1374         return true;
1375     }
1376 
1377     /**
1378      * If a refresh has been specified either through a meta tag or an HTTP
1379      * response header, then perform that refresh.
1380      * @throws IOException if an IO problem occurs
1381      */
1382     private void executeRefreshIfNeeded() throws IOException {
1383         // If this page is not in a frame then a refresh has already happened,
1384         // most likely through the JavaScript onload handler, so we don't do a
1385         // second refresh.
1386         final WebWindow window = getEnclosingWindow();
1387         if (window == null) {
1388             return;
1389         }
1390 
1391         final String refreshString = getRefreshStringOrNull();
1392         if (refreshString == null || refreshString.isEmpty()) {
1393             return;
1394         }
1395 
1396         final double time;
1397         final URL url;
1398 
1399         final int index = StringUtils.indexOfAnyBut(refreshString, "0123456789.");
1400 
1401         if (index == -1) {
1402             // Format: <meta http-equiv='refresh' content='10'>
1403             try {
1404                 time = Double.parseDouble(refreshString);
1405             }
1406             catch (final NumberFormatException e) {
1407                 if (LOG.isErrorEnabled()) {
1408                     LOG.error("Malformed refresh string (no ';' but not a number): " + refreshString, e);
1409                 }
1410                 return;
1411             }
1412             url = getUrl();
1413         }
1414         else {
1415             // Format: <meta http-equiv='refresh' content='10;url=http://www.blah.com'>
1416             try {
1417                 time = Double.parseDouble(refreshString.substring(0, index));
1418             }
1419             catch (final NumberFormatException e) {
1420                 if (LOG.isErrorEnabled()) {
1421                     LOG.error("Malformed refresh string (no valid number before ';') " + refreshString, e);
1422                 }
1423                 return;
1424             }
1425 
1426             String urlPart = refreshString.substring(index);
1427             final char separator = urlPart.charAt(0);
1428             if (";, \r\n\t".indexOf(separator) >= 0) {
1429                 urlPart = StringUtils.stripStart(urlPart, ";, \r\n\t");
1430                 urlPart = urlPart.toLowerCase(Locale.ROOT);
1431                 if (urlPart.startsWith("url")) {
1432                     urlPart = urlPart.substring(3);
1433                     urlPart = urlPart.trim();
1434 
1435                     if (urlPart.length() > 0 && urlPart.charAt(0) == '=') {
1436                         urlPart = urlPart.substring(1);
1437                         urlPart = urlPart.trim();
1438                     }
1439                 }
1440 
1441                 if (org.htmlunit.util.StringUtils.isBlank(urlPart)) {
1442                     //content='10; URL=' is treated as content='10'
1443                     url = getUrl();
1444                 }
1445                 else {
1446                     if (urlPart.charAt(0) == '"' || urlPart.charAt(0) == 0x27) {
1447                         urlPart = urlPart.substring(1);
1448                     }
1449                     if (urlPart.charAt(urlPart.length() - 1) == '"' || urlPart.charAt(urlPart.length() - 1) == 0x27) {
1450                         urlPart = urlPart.substring(0, urlPart.length() - 1);
1451                     }
1452                     try {
1453                         url = getFullyQualifiedUrl(urlPart);
1454                     }
1455                     catch (final MalformedURLException e) {
1456                         if (LOG.isErrorEnabled()) {
1457                             LOG.error("Malformed URL in refresh string: " + refreshString, e);
1458                         }
1459                         return;
1460                     }
1461                 }
1462             }
1463             else {
1464                 if (LOG.isErrorEnabled()) {
1465                     LOG.error("Malformed refresh string (separator after time missing): " + refreshString);
1466                 }
1467                 return;
1468             }
1469         }
1470 
1471         processRefresh(url, time);
1472     }
1473 
1474     // this is different from what is done in org.htmlunit.WebClient.loadWebResponseFromWebConnection(WebRequest, int)
1475     // because there we are directly replacing the response before loading the response into the window
1476     // here we are replacing the page in the window (maybe after some time)
1477     private void processRefresh(final URL url, final double time) throws IOException {
1478         final WebClient webClient = getWebClient();
1479 
1480         final int refreshLimit = webClient.getOptions().getPageRefreshLimit();
1481         if (refreshLimit == 0) {
1482             final WebResponse webResponse = getWebResponse();
1483             throw new FailingHttpStatusCodeException("Too many redirects for "
1484                     + webResponse.getWebRequest().getUrl(), webResponse);
1485         }
1486 
1487         if (refreshLimit >= 0) {
1488             final StackTraceElement[] elements = new Exception().getStackTrace();
1489             int count = 0;
1490             final int elementCountLimit = refreshLimit > 50 ? 400 : refreshLimit > 10 ? 80 : 5;
1491             final int elementCount = elements.length;
1492 
1493             if (elementCount > elementCountLimit) {
1494                 for (int i = 0; i < elementCount; i++) {
1495                     if ("processRefresh".equals(elements[i].getMethodName())
1496                             && "org.htmlunit.html.HtmlPage".equals(elements[i].getClassName())) {
1497                         count++;
1498                         if (count >= refreshLimit) {
1499                             final WebResponse webResponse = getWebResponse();
1500                             throw new FailingHttpStatusCodeException(
1501                                             "Too many redirects (>= " + count + ") for "
1502                                                 + webResponse.getWebRequest().getUrl(), webResponse);
1503                         }
1504                     }
1505                 }
1506             }
1507         }
1508 
1509         webClient.getRefreshHandler().handleRefresh(this, url, (int) time);
1510     }
1511 
1512     /**
1513      * Returns an auto-refresh string if specified. This will look in both the meta
1514      * tags and inside the HTTP response headers.
1515      * @return the auto-refresh string
1516      */
1517     private String getRefreshStringOrNull() {
1518         final List<HtmlMeta> metaTags = getMetaTags("refresh");
1519         if (!metaTags.isEmpty()) {
1520             return metaTags.get(0).getContentAttribute().trim();
1521         }
1522         return getWebResponse().getResponseHeaderValue("Refresh");
1523     }
1524 
1525     private void processPostponedActionsIfNeeded() {
1526         if (!getWebClient().isJavaScriptEnabled()) {
1527             return;
1528         }
1529         getWebClient().getJavaScriptEngine().processPostponedActions();
1530     }
1531 
1532     /**
1533      * Executes any deferred scripts, if necessary.
1534      */
1535     private void executeDeferredScriptsIfNeeded() {
1536         if (!getWebClient().isJavaScriptEnabled()) {
1537             return;
1538         }
1539         final DomElement doc = getDocumentElement();
1540         final List<HtmlScript> scripts = new ArrayList<>();
1541 
1542         // don't call getElementsByTagName() here because it creates a live collection
1543         for (final HtmlElement elem : doc.getHtmlElementDescendants()) {
1544             if ("script".equals(elem.getLocalName()) && (elem instanceof HtmlScript script)) {
1545                 if (script.isDeferred() && ATTRIBUTE_NOT_DEFINED != script.getSrcAttribute()) {
1546                     scripts.add(script);
1547                 }
1548             }
1549         }
1550         for (final HtmlScript script : scripts) {
1551             ScriptElementSupport.executeScriptIfNeeded(script, true, true);
1552         }
1553     }
1554 
1555     /**
1556      * Deregister frames that are no longer in use.
1557      */
1558     public void deregisterFramesIfNeeded() {
1559         final List<BaseFrameElement> frameElementsCopy = new ArrayList<>(frameElements_);
1560         for (final BaseFrameElement frameElement : frameElementsCopy) {
1561             final WebWindow window = frameElement.getEnclosedWindow();
1562             getWebClient().deregisterWebWindow(window);
1563             final Page page = window.getEnclosedPage();
1564             if (page != null && page.isHtmlPage()) {
1565                 // seems quite silly, but for instance if the src attribute of an iframe is not
1566                 // set, the error only occurs when leaving the page
1567                 ((HtmlPage) page).deregisterFramesIfNeeded();
1568             }
1569         }
1570     }
1571 
1572     /**
1573      * Returns a list containing all the frames (from frame and iframe tags) in this page
1574      * in document order.
1575      * @return a list of {@link FrameWindow}
1576      */
1577     public List<FrameWindow> getFrames() {
1578         final List<BaseFrameElement> frameElements = new ArrayList<>(frameElements_);
1579         frameElements.sort(DOCUMENT_POSITION_COMPERATOR);
1580 
1581         final List<FrameWindow> list = new ArrayList<>(frameElements.size());
1582         for (final BaseFrameElement frameElement : frameElements) {
1583             list.add(frameElement.getEnclosedWindow());
1584         }
1585         return list;
1586     }
1587 
1588     /**
1589      * Returns the first frame contained in this page with the specified name.
1590      * @param name the name to search for
1591      * @return the first frame found
1592      * @throws ElementNotFoundException If no frame exist in this page with the specified name.
1593      */
1594     public FrameWindow getFrameByName(final String name) throws ElementNotFoundException {
1595         for (final BaseFrameElement frameElement : frameElements_) {
1596             final FrameWindow fw = frameElement.getEnclosedWindow();
1597             if (fw.getName().equals(name)) {
1598                 return fw;
1599             }
1600         }
1601 
1602         throw new ElementNotFoundException("frame or iframe", DomElement.NAME_ATTRIBUTE, name);
1603     }
1604 
1605     /**
1606      * Simulate pressing an access key. This may change the focus, may click buttons and may invoke
1607      * JavaScript.
1608      *
1609      * @param accessKey the key that will be pressed
1610      * @return the element that has the focus after pressing this access key or null if no element
1611      *         has the focus.
1612      * @throws IOException if an IO error occurs during the processing of this access key (this
1613      *         would only happen if the access key triggered a button which in turn caused a page load)
1614      */
1615     public DomElement pressAccessKey(final char accessKey) throws IOException {
1616         final HtmlElement element = getHtmlElementByAccessKey(accessKey);
1617         if (element != null) {
1618             element.focus();
1619             if (element instanceof HtmlAnchor
1620                     || element instanceof HtmlArea
1621                     || element instanceof HtmlButton
1622                     || element instanceof HtmlInput
1623                     || element instanceof HtmlLabel
1624                     || element instanceof HtmlLegend
1625                     || element instanceof HtmlTextArea) {
1626                 final Page newPage = element.click();
1627 
1628                 if (newPage != this && getFocusedElement() == element) {
1629                     // The page was reloaded therefore no element on this page will have the focus.
1630                     getFocusedElement().blur();
1631                 }
1632             }
1633         }
1634 
1635         return getFocusedElement();
1636     }
1637 
1638     /**
1639      * Move the focus to the next element in the tab order. To determine the specified tab
1640      * order, refer to {@link HtmlPage#getTabbableElements()}
1641      *
1642      * @return the element that has focus after calling this method
1643      */
1644     public HtmlElement tabToNextElement() {
1645         final List<HtmlElement> elements = getTabbableElements();
1646         if (elements.isEmpty()) {
1647             setFocusedElement(null);
1648             return null;
1649         }
1650 
1651         final HtmlElement elementToGiveFocus;
1652         final DomElement elementWithFocus = getFocusedElement();
1653         if (elementWithFocus == null) {
1654             elementToGiveFocus = elements.get(0);
1655         }
1656         else {
1657             final int index = elements.indexOf(elementWithFocus);
1658             if (index == -1) {
1659                 // The element with focus isn't on this page
1660                 elementToGiveFocus = elements.get(0);
1661             }
1662             else if (index == elements.size() - 1) {
1663                 // if at last jump to start
1664                 elementToGiveFocus = elements.get(0);
1665             }
1666             else {
1667                 elementToGiveFocus = elements.get(index + 1);
1668             }
1669         }
1670 
1671         setFocusedElement(elementToGiveFocus);
1672         return elementToGiveFocus;
1673     }
1674 
1675     /**
1676      * Move the focus to the previous element in the tab order. To determine the specified tab
1677      * order, refer to {@link HtmlPage#getTabbableElements()}
1678      *
1679      * @return the element that has focus after calling this method
1680      */
1681     public HtmlElement tabToPreviousElement() {
1682         final List<HtmlElement> elements = getTabbableElements();
1683         if (elements.isEmpty()) {
1684             setFocusedElement(null);
1685             return null;
1686         }
1687 
1688         final HtmlElement elementToGiveFocus;
1689         final DomElement elementWithFocus = getFocusedElement();
1690         if (elementWithFocus == null) {
1691             elementToGiveFocus = elements.get(elements.size() - 1);
1692         }
1693         else {
1694             final int index = elements.indexOf(elementWithFocus);
1695             if (index == -1) {
1696                 // The element with focus isn't on this page
1697                 elementToGiveFocus = elements.get(elements.size() - 1);
1698             }
1699             else if (index == 0) {
1700                 // first; back to the last
1701                 elementToGiveFocus = elements.get(elements.size() - 1);
1702             }
1703             else {
1704                 elementToGiveFocus = elements.get(index - 1);
1705             }
1706         }
1707 
1708         setFocusedElement(elementToGiveFocus);
1709         return elementToGiveFocus;
1710     }
1711 
1712     /**
1713      * Returns the HTML element with the specified ID. If more than one element
1714      * has this ID (not allowed by the HTML spec), then this method returns the
1715      * first one.
1716      *
1717      * @param elementId the ID value to search for
1718      * @param <E> the element type
1719      * @return the HTML element with the specified ID
1720      * @throws ElementNotFoundException if no element was found matching the specified ID
1721      */
1722     @SuppressWarnings("unchecked")
1723     public <E extends HtmlElement> E getHtmlElementById(final String elementId) throws ElementNotFoundException {
1724         final DomElement element = getElementById(elementId);
1725         if (element == null) {
1726             throw new ElementNotFoundException("*", DomElement.ID_ATTRIBUTE, elementId);
1727         }
1728         return (E) element;
1729     }
1730 
1731     /**
1732      * Returns the elements with the specified ID. If there are no elements
1733      * with the specified ID, this method returns an empty list. Please note that
1734      * the lists returned by this method are immutable.
1735      *
1736      * @param elementId the ID value to search for
1737      * @return the elements with the specified name attribute
1738      */
1739     public List<DomElement> getElementsById(final String elementId) {
1740         if (elementId != null) {
1741             ensureMappedElementsBuilt();
1742             final MappedElementIndexEntry elements = idMap_.get(elementId);
1743             if (elements != null) {
1744                 return new ArrayList<>(elements.elements());
1745             }
1746         }
1747         return Collections.emptyList();
1748     }
1749 
1750     /**
1751      * Returns the element with the specified name. If more than one element
1752      * has this name, then this method returns the first one.
1753      *
1754      * @param name the name value to search for
1755      * @param <E> the element type
1756      * @return the element with the specified name
1757      * @throws ElementNotFoundException if no element was found matching the specified name
1758      */
1759     @SuppressWarnings("unchecked")
1760     public <E extends DomElement> E getElementByName(final String name) throws ElementNotFoundException {
1761         if (name != null) {
1762             ensureMappedElementsBuilt();
1763             final MappedElementIndexEntry elements = nameMap_.get(name);
1764             if (elements != null) {
1765                 return (E) elements.first();
1766             }
1767         }
1768         throw new ElementNotFoundException("*", DomElement.NAME_ATTRIBUTE, name);
1769     }
1770 
1771     /**
1772      * Returns the elements with the specified name attribute. If there are no elements
1773      * with the specified name, this method returns an empty list. Please note that
1774      * the lists returned by this method are immutable.
1775      *
1776      * @param name the name value to search for
1777      * @return the elements with the specified name attribute
1778      */
1779     public List<DomElement> getElementsByName(final String name) {
1780         if (name != null) {
1781             ensureMappedElementsBuilt();
1782             final MappedElementIndexEntry elements = nameMap_.get(name);
1783             if (elements != null) {
1784                 return new ArrayList<>(elements.elements());
1785             }
1786         }
1787         return Collections.emptyList();
1788     }
1789 
1790     /**
1791      * Returns the elements with the specified string for their name or ID. If there are
1792      * no elements with the specified name or ID, this method returns an empty list.
1793      *
1794      * @param idAndOrName the value to search for
1795      * @return the elements with the specified string for their name or ID
1796      */
1797     public List<DomElement> getElementsByIdAndOrName(final String idAndOrName) {
1798         if (idAndOrName == null) {
1799             return Collections.emptyList();
1800         }
1801         ensureMappedElementsBuilt();
1802         final MappedElementIndexEntry list1 = idMap_.get(idAndOrName);
1803         final MappedElementIndexEntry list2 = nameMap_.get(idAndOrName);
1804         final List<DomElement> list = new ArrayList<>();
1805         if (list1 != null) {
1806             list.addAll(list1.elements());
1807         }
1808         if (list2 != null) {
1809             for (final DomElement elt : list2.elements()) {
1810                 if (!list.contains(elt)) {
1811                     list.add(elt);
1812                 }
1813             }
1814         }
1815         return list;
1816     }
1817 
1818     /**
1819      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1820      *
1821      * @param node the node that has just been added to the document
1822      */
1823     void notifyNodeAdded(final DomNode node) {
1824         if (node instanceof DomElement element1) {
1825             addMappedElement(element1, true);
1826 
1827             if (node instanceof BaseFrameElement element) {
1828                 frameElements_.add(element);
1829             }
1830 
1831             if (node.getFirstChild() != null) {
1832                 for (final Iterator<HtmlElement> iterator = node.new DescendantHtmlElementsIterator();
1833                         iterator.hasNext();) {
1834                     final HtmlElement child = iterator.next();
1835                     if (child instanceof BaseFrameElement element) {
1836                         frameElements_.add(element);
1837                     }
1838                 }
1839             }
1840 
1841             if ("base".equals(node.getNodeName())) {
1842                 calculateBase();
1843             }
1844         }
1845         node.onAddedToPage();
1846     }
1847 
1848     /**
1849      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1850      *
1851      * @param node the node that has just been removed from the tree
1852      */
1853     void notifyNodeRemoved(final DomNode node) {
1854         if (node instanceof HtmlElement element) {
1855             removeMappedElement(element, true, true);
1856 
1857             if (node instanceof BaseFrameElement) {
1858                 frameElements_.remove(node);
1859             }
1860             for (final HtmlElement child : node.getHtmlElementDescendants()) {
1861                 if (child instanceof BaseFrameElement) {
1862                     frameElements_.remove(child);
1863                 }
1864             }
1865 
1866             if ("base".equals(node.getNodeName())) {
1867                 calculateBase();
1868             }
1869         }
1870     }
1871 
1872     /**
1873      * Adds an element to the ID and name maps, if necessary.
1874      * @param element the element to be added to the ID and name maps
1875      * @param recurse indicates if children must be added too
1876      */
1877     void addMappedElement(final DomElement element, final boolean recurse) {
1878         // Index is built lazily; skip while not built. ensureMappedElementsBuilt()
1879         // walks the tree once and populates everything on first read.
1880         if (!mappedElementsBuilt_) {
1881             return;
1882         }
1883         if (isAncestorOf(element)) {
1884             addElement(element, recurse);
1885         }
1886     }
1887 
1888     private void ensureMappedElementsBuilt() {
1889         if (mappedElementsBuilt_) {
1890             return;
1891         }
1892 
1893         final DomElement root = getDocumentElement();
1894         if (root != null) {
1895             addElement(root, true);
1896         }
1897 
1898         // Flip the flag only after the maps are populated, so a partial
1899         // failure mid-walk leaves us with built_=false and the next read
1900         // tries again rather than seeing a half-populated index.
1901         mappedElementsBuilt_ = true;
1902     }
1903 
1904     private void addElement(final DomElement element, final boolean recurse) {
1905         final String idValue = element.getAttribute(DomElement.ID_ATTRIBUTE);
1906         if (ATTRIBUTE_NOT_DEFINED != idValue) {
1907             MappedElementIndexEntry elements = idMap_.get(idValue);
1908             if (elements == null) {
1909                 elements = new MappedElementIndexEntry();
1910                 elements.add(element);
1911                 idMap_.put(idValue, elements);
1912             }
1913             else {
1914                 elements.add(element);
1915             }
1916         }
1917 
1918         final String nameValue = element.getAttribute(DomElement.NAME_ATTRIBUTE);
1919         if (ATTRIBUTE_NOT_DEFINED != nameValue) {
1920             MappedElementIndexEntry elements = nameMap_.get(nameValue);
1921             if (elements == null) {
1922                 elements = new MappedElementIndexEntry();
1923                 elements.add(element);
1924                 nameMap_.put(nameValue, elements);
1925             }
1926             else {
1927                 elements.add(element);
1928             }
1929         }
1930 
1931         if (recurse) {
1932             // poor man's approach - we don't use getChildElements()
1933             // to avoid a bunch of object constructions
1934             DomNode nextChild = element.getFirstChild();
1935             while (nextChild != null) {
1936                 if (nextChild instanceof DomElement domElement) {
1937                     addElement(domElement, true);
1938                 }
1939                 nextChild = nextChild.getNextSibling();
1940             }
1941         }
1942     }
1943 
1944     /**
1945      * Removes an element and optionally its children from the ID and name maps, if necessary.
1946      * @param element the element to be removed from the ID and name maps
1947      * @param recurse indicates if children must be removed too
1948      * @param descendant indicates of the element was descendant of this HtmlPage, but now its parent might be null
1949      */
1950     void removeMappedElement(final DomElement element, final boolean recurse, final boolean descendant) {
1951         // see addMappedElement: while the index is unbuilt, removals are also no-ops.
1952         if (!mappedElementsBuilt_) {
1953             return;
1954         }
1955         if (descendant || isAncestorOf(element)) {
1956             removeElement(element, recurse);
1957         }
1958     }
1959 
1960     private void removeElement(final DomElement element, final boolean recurse) {
1961         final String idValue = element.getAttribute(DomElement.ID_ATTRIBUTE);
1962         if (ATTRIBUTE_NOT_DEFINED != idValue) {
1963             final MappedElementIndexEntry elements = idMap_.remove(idValue);
1964             if (elements != null) {
1965                 elements.remove(element);
1966                 if (!elements.elements_.isEmpty()) {
1967                     idMap_.put(idValue, elements);
1968                 }
1969             }
1970         }
1971 
1972         final String nameValue = element.getAttribute(DomElement.NAME_ATTRIBUTE);
1973         if (ATTRIBUTE_NOT_DEFINED != nameValue) {
1974             final MappedElementIndexEntry elements = nameMap_.remove(nameValue);
1975             if (elements != null) {
1976                 elements.remove(element);
1977                 if (!elements.elements_.isEmpty()) {
1978                     nameMap_.put(nameValue, elements);
1979                 }
1980             }
1981         }
1982 
1983         if (recurse) {
1984             for (final DomElement child : element.getChildElements()) {
1985                 removeElement(child, true);
1986             }
1987         }
1988     }
1989 
1990     /**
1991      * Indicates if the attribute name indicates that the owning element is mapped.
1992      * @param document the owning document
1993      * @param attributeName the name of the attribute to consider
1994      * @return {@code true} if the owning element should be mapped in its owning page
1995      */
1996     static boolean isMappedElement(final Document document, final String attributeName) {
1997         return document instanceof HtmlPage
1998             && (DomElement.NAME_ATTRIBUTE.equals(attributeName) || DomElement.ID_ATTRIBUTE.equals(attributeName));
1999     }
2000 
2001     private void calculateBase() {
2002         final List<HtmlElement> baseElements = getDocumentElement().getStaticElementsByTagName("base");
2003 
2004         base_ = null;
2005         for (final HtmlElement baseElement : baseElements) {
2006             if (baseElement instanceof HtmlBase base) {
2007                 if (base_ != null) {
2008                     notifyIncorrectness("Multiple 'base' detected, only the first is used.");
2009                     break;
2010                 }
2011                 base_ = base;
2012             }
2013         }
2014     }
2015 
2016     /**
2017      * Loads the content of the contained frames. This is done after the page is completely loaded, to allow script
2018      * contained in the frames to reference elements from the page located after the closing &lt;/frame&gt; tag.
2019      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
2020      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to {@code true}
2021      */
2022     void loadFrames() throws FailingHttpStatusCodeException {
2023         for (final BaseFrameElement frameElement : new ArrayList<>(frameElements_)) {
2024             // test if the frame should really be loaded:
2025             // if a script has already changed its content, it should be skipped
2026             // use == and not equals(...) to identify initial content (versus URL set to "about:blank")
2027             if (frameElement.getEnclosedWindow() != null
2028                     && UrlUtils.URL_ABOUT_BLANK == frameElement.getEnclosedPage().getUrl()
2029                     && !frameElement.isContentLoaded()) {
2030                 frameElement.loadInnerPage();
2031             }
2032         }
2033     }
2034 
2035     /**
2036      * Gives a basic representation for debugging purposes.
2037      * @return a basic representation
2038      */
2039     @Override
2040     public String toString() {
2041         final StringBuilder builder = new StringBuilder()
2042             .append("HtmlPage(")
2043             .append(getUrl())
2044             .append(")@")
2045             .append(hashCode());
2046         return builder.toString();
2047     }
2048 
2049     /**
2050      * Gets the meta tag for a given {@code http-equiv} value.
2051      * @param httpEquiv the {@code http-equiv} value
2052      * @return a list of {@link HtmlMeta}
2053      */
2054     protected List<HtmlMeta> getMetaTags(final String httpEquiv) {
2055         if (getDocumentElement() == null) {
2056             return Collections.emptyList(); // weird case, for instance if document.documentElement has been removed
2057         }
2058         final List<HtmlMeta> tags = getDocumentElement().getStaticElementsByTagName("meta");
2059         final List<HtmlMeta> foundTags = new ArrayList<>();
2060         for (final HtmlMeta htmlMeta : tags) {
2061             if (httpEquiv.equalsIgnoreCase(htmlMeta.getHttpEquivAttribute())) {
2062                 foundTags.add(htmlMeta);
2063             }
2064         }
2065         return foundTags;
2066     }
2067 
2068     /**
2069      * Creates a clone of this instance, and clears cached state to be not shared with the original.
2070      *
2071      * @return a clone of this instance
2072      */
2073     @Override
2074     protected HtmlPage clone() {
2075         final HtmlPage result = (HtmlPage) super.clone();
2076         result.elementWithFocus_ = null;
2077 
2078         result.idMap_ = new ConcurrentHashMap<>();
2079         result.nameMap_ = new ConcurrentHashMap<>();
2080         result.mappedElementsBuilt_ = false;
2081 
2082         return result;
2083     }
2084 
2085     /**
2086      * {@inheritDoc}
2087      */
2088     @Override
2089     public HtmlPage cloneNode(final boolean deep) {
2090         // we need the ScriptObject clone before cloning the kids.
2091         final HtmlPage result = (HtmlPage) super.cloneNode(false);
2092         if (getWebClient().isJavaScriptEnabled()) {
2093             final HtmlUnitScriptable jsObjClone = getScriptableObject().clone();
2094             jsObjClone.setDomNode(result);
2095         }
2096 
2097         // if deep, clone the kids too, and re initialize parts of the clone
2098         if (deep) {
2099             // this was previously synchronized but that makes not sense, why
2100             // lock the source against a copy only one has a reference too,
2101             // because result is a local reference
2102             result.attributeListeners_ = null;
2103 
2104             result.selectionRanges_ = new ArrayList<>(3);
2105             // the original one is synchronized so we should do that here too, shouldn't we?
2106             result.afterLoadActions_ = Collections.synchronizedList(new ArrayList<>());
2107             result.frameElements_ = new ArrayList<>();
2108             for (DomNode child = getFirstChild(); child != null; child = child.getNextSibling()) {
2109                 result.appendChild(child.cloneNode(true));
2110             }
2111         }
2112         return result;
2113     }
2114 
2115     /**
2116      * Adds an HtmlAttributeChangeListener to the listener list.
2117      * The listener is registered for all attributes of all HtmlElements contained in this page.
2118      *
2119      * @param listener the attribute change listener to be added
2120      * @see #removeHtmlAttributeChangeListener(HtmlAttributeChangeListener)
2121      */
2122     public void addHtmlAttributeChangeListener(final HtmlAttributeChangeListener listener) {
2123         WebAssert.notNull("listener", listener);
2124         synchronized (lock_) {
2125             if (attributeListeners_ == null) {
2126                 attributeListeners_ = new LinkedHashSet<>();
2127             }
2128             attributeListeners_.add(listener);
2129         }
2130     }
2131 
2132     /**
2133      * Removes an HtmlAttributeChangeListener from the listener list.
2134      * This method should be used to remove HtmlAttributeChangeListener that were registered
2135      * for all attributes of all HtmlElements contained in this page.
2136      *
2137      * @param listener the attribute change listener to be removed
2138      * @see #addHtmlAttributeChangeListener(HtmlAttributeChangeListener)
2139      */
2140     public void removeHtmlAttributeChangeListener(final HtmlAttributeChangeListener listener) {
2141         WebAssert.notNull("listener", listener);
2142         synchronized (lock_) {
2143             if (attributeListeners_ != null) {
2144                 attributeListeners_.remove(listener);
2145             }
2146         }
2147     }
2148 
2149     /**
2150      * Notifies all registered listeners for the given event to add an attribute.
2151      * @param event the event to fire
2152      */
2153     void fireHtmlAttributeAdded(final HtmlAttributeChangeEvent event) {
2154         final List<HtmlAttributeChangeListener> listeners = safeGetAttributeListeners();
2155         if (listeners != null) {
2156             for (final HtmlAttributeChangeListener listener : listeners) {
2157                 listener.attributeAdded(event);
2158             }
2159         }
2160     }
2161 
2162     /**
2163      * Notifies all registered listeners for the given event to replace an attribute.
2164      * @param event the event to fire
2165      */
2166     void fireHtmlAttributeReplaced(final HtmlAttributeChangeEvent event) {
2167         final List<HtmlAttributeChangeListener> listeners = safeGetAttributeListeners();
2168         if (listeners != null) {
2169             for (final HtmlAttributeChangeListener listener : listeners) {
2170                 listener.attributeReplaced(event);
2171             }
2172         }
2173     }
2174 
2175     /**
2176      * Notifies all registered listeners for the given event to remove an attribute.
2177      * @param event the event to fire
2178      */
2179     void fireHtmlAttributeRemoved(final HtmlAttributeChangeEvent event) {
2180         final List<HtmlAttributeChangeListener> listeners = safeGetAttributeListeners();
2181         if (listeners != null) {
2182             for (final HtmlAttributeChangeListener listener : listeners) {
2183                 listener.attributeRemoved(event);
2184             }
2185         }
2186     }
2187 
2188     private List<HtmlAttributeChangeListener> safeGetAttributeListeners() {
2189         synchronized (lock_) {
2190             if (attributeListeners_ != null) {
2191                 return new ArrayList<>(attributeListeners_);
2192             }
2193             return null;
2194         }
2195     }
2196 
2197     /**
2198      * {@inheritDoc}
2199      */
2200     @Override
2201     protected void checkChildHierarchy(final org.w3c.dom.Node newChild) throws DOMException {
2202         if (newChild instanceof Element) {
2203             if (getDocumentElement() != null) {
2204                 throw new DOMException(DOMException.HIERARCHY_REQUEST_ERR,
2205                     "The Document may only have a single child Element.");
2206             }
2207         }
2208         else if (newChild instanceof DocumentType) {
2209             if (getDoctype() != null) {
2210                 throw new DOMException(DOMException.HIERARCHY_REQUEST_ERR,
2211                     "The Document may only have a single child DocumentType.");
2212             }
2213         }
2214         else if (!(newChild instanceof Comment || newChild instanceof ProcessingInstruction)) {
2215             throw new DOMException(DOMException.HIERARCHY_REQUEST_ERR,
2216                 "The Document may not have a child of this type: " + newChild.getNodeType());
2217         }
2218         super.checkChildHierarchy(newChild);
2219     }
2220 
2221     /**
2222      * Returns {@code true} if an HTML parser is operating on this page, adding content to it.
2223      * @return {@code true} if an HTML parser is operating on this page, adding content to it
2224      */
2225     public boolean isBeingParsed() {
2226         return parserCount_ > 0;
2227     }
2228 
2229     /**
2230      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2231      *
2232      * Called by the HTML parser to let the page know that it has started parsing some content for this page.
2233      */
2234     public void registerParsingStart() {
2235         parserCount_++;
2236     }
2237 
2238     /**
2239      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2240      *
2241      * Called by the HTML parser to let the page know that it has finished parsing some content for this page.
2242      */
2243     public void registerParsingEnd() {
2244         parserCount_--;
2245     }
2246 
2247     /**
2248      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2249      *
2250      * Returns {@code true} if an HTML parser is parsing a non-inline HTML snippet to add content
2251      * to this page. Non-inline content is content that is parsed for the page, but not in the
2252      * same stream as the page itself -- basically anything other than <code>document.write()</code>
2253      * or <code>document.writeln()</code>: <code>innerHTML</code>, <code>outerHTML</code>,
2254      * <code>document.createElement()</code>, etc.
2255      *
2256      * @return {@code true} if an HTML parser is parsing a non-inline HTML snippet to add content
2257      *         to this page
2258      */
2259     public boolean isParsingHtmlSnippet() {
2260         return snippetParserCount_ > 0;
2261     }
2262 
2263     /**
2264      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2265      *
2266      * Called by the HTML parser to let the page know that it has started parsing a non-inline HTML snippet.
2267      */
2268     public void registerSnippetParsingStart() {
2269         snippetParserCount_++;
2270     }
2271 
2272     /**
2273      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2274      *
2275      * Called by the HTML parser to let the page know that it has finished parsing a non-inline HTML snippet.
2276      */
2277     public void registerSnippetParsingEnd() {
2278         snippetParserCount_--;
2279     }
2280 
2281     /**
2282      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2283      *
2284      * Returns {@code true} if an HTML parser is parsing an inline HTML snippet to add content
2285      * to this page. Inline content is content inserted into the parser stream dynamically
2286      * while the page is being parsed (i.e. <code>document.write()</code> or <code>document.writeln()</code>).
2287      *
2288      * @return {@code true} if an HTML parser is parsing an inline HTML snippet to add content
2289      *         to this page
2290      */
2291     public boolean isParsingInlineHtmlSnippet() {
2292         return inlineSnippetParserCount_ > 0;
2293     }
2294 
2295     /**
2296      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2297      *
2298      * Called by the HTML parser to let the page know that it has started parsing an inline HTML snippet.
2299      */
2300     public void registerInlineSnippetParsingStart() {
2301         inlineSnippetParserCount_++;
2302     }
2303 
2304     /**
2305      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2306      *
2307      * Called by the HTML parser to let the page know that it has finished parsing an inline HTML snippet.
2308      */
2309     public void registerInlineSnippetParsingEnd() {
2310         inlineSnippetParserCount_--;
2311     }
2312 
2313     /**
2314      * Refreshes the page by sending the same parameters as previously sent to get this page.
2315      * @return the newly loaded page.
2316      * @throws IOException if an IO problem occurs
2317      */
2318     public Page refresh() throws IOException {
2319         return getWebClient().getPage(getWebResponse().getWebRequest());
2320     }
2321 
2322     /**
2323      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2324      * <p>
2325      * Parses the given string as would it belong to the content being parsed
2326      * at the current parsing position
2327      * </p>
2328      * @param string the HTML code to write in place
2329      */
2330     public void writeInParsedStream(final String string) {
2331         getDOMBuilder().pushInputString(string);
2332     }
2333 
2334     /**
2335      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2336      *
2337      * Sets the builder to allow page to send content from document.write(ln) calls.
2338      * @param htmlUnitDOMBuilder the builder
2339      */
2340     public void setDOMBuilder(final HTMLParserDOMBuilder htmlUnitDOMBuilder) {
2341         domBuilder_ = htmlUnitDOMBuilder;
2342     }
2343 
2344     /**
2345      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2346      *
2347      * Returns the current builder.
2348      * @return the current builder
2349      */
2350     public HTMLParserDOMBuilder getDOMBuilder() {
2351         return domBuilder_;
2352     }
2353 
2354     /**
2355      * <p>Returns all namespaces defined in the root element of this page.</p>
2356      * <p>The default namespace has a key of an empty string.</p>
2357      * @return all namespaces defined in the root element of this page
2358      */
2359     public Map<String, String> getNamespaces() {
2360         final org.w3c.dom.NamedNodeMap attributes = getDocumentElement().getAttributes();
2361         final Map<String, String> namespaces = new HashMap<>();
2362         for (int i = 0; i < attributes.getLength(); i++) {
2363             final Attr attr = (Attr) attributes.item(i);
2364             String name = attr.getName();
2365             if (name.startsWith("xmlns")) {
2366                 int startPos = 5;
2367                 if (name.length() > 5 && name.charAt(5) == ':') {
2368                     startPos = 6;
2369                 }
2370                 name = name.substring(startPos);
2371                 namespaces.put(name, attr.getValue());
2372             }
2373         }
2374         return namespaces;
2375     }
2376 
2377     /**
2378      * {@inheritDoc}
2379      */
2380     @Override
2381     public void setDocumentType(final DocumentType type) {
2382         super.setDocumentType(type);
2383     }
2384 
2385     /**
2386      * Saves the current page, with all images, to the specified location.
2387      * The default behavior removes all script elements.
2388      *
2389      * @param file file to write this page into
2390      * @throws IOException If an error occurs
2391      */
2392     public void save(final File file) throws IOException {
2393         new XmlSerializer().save(this, file);
2394     }
2395 
2396     /**
2397      * Returns whether the current page mode is in {@code quirks mode} or in {@code standards mode}.
2398      * @return true for {@code quirks mode}, false for {@code standards mode}
2399      */
2400     public boolean isQuirksMode() {
2401         return "BackCompat".equals(((HTMLDocument) getScriptableObject()).getCompatMode());
2402     }
2403 
2404     /**
2405      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2406      * {@inheritDoc}
2407      */
2408     @Override
2409     public boolean isAttachedToPage() {
2410         return true;
2411     }
2412 
2413     /**
2414      * {@inheritDoc}
2415      */
2416     @Override
2417     public boolean isHtmlPage() {
2418         return true;
2419     }
2420 
2421     /**
2422      * The base URL used to resolve relative URLs.
2423      * @return the base URL
2424      */
2425     public URL getBaseURL() {
2426         URL baseUrl;
2427         if (base_ == null) {
2428             baseUrl = getUrl();
2429             final WebWindow window = getEnclosingWindow();
2430             final boolean frame = window != null && window != window.getTopWindow();
2431             if (frame) {
2432                 final boolean frameSrcIsNotSet = baseUrl == UrlUtils.URL_ABOUT_BLANK;
2433                 final boolean frameSrcIsJs = "javascript".equals(baseUrl.getProtocol());
2434                 if (frameSrcIsNotSet || frameSrcIsJs) {
2435                     baseUrl = window.getTopWindow().getEnclosedPage().getWebResponse()
2436                         .getWebRequest().getUrl();
2437                 }
2438             }
2439             else if (baseUrl_ != null) {
2440                 baseUrl = baseUrl_;
2441             }
2442         }
2443         else {
2444             final String href = base_.getHrefAttribute().trim();
2445             if (org.htmlunit.util.StringUtils.isEmptyOrNull(href)) {
2446                 baseUrl = getUrl();
2447             }
2448             else {
2449                 final URL url = getUrl();
2450                 try {
2451                     if (href.startsWith("http://") || href.startsWith("https://")) {
2452                         baseUrl = new URL(href);
2453                     }
2454                     else if (href.startsWith("//")) {
2455                         baseUrl = new URL("%s:%s".formatted(url.getProtocol(), href));
2456                     }
2457                     else if (href.length() > 0 && href.charAt(0) == '/') {
2458                         final int port = Window.getPort(url);
2459                         baseUrl = new URL("%s://%s:%d%s".formatted(url.getProtocol(), url.getHost(), port, href));
2460                     }
2461                     else if (url.toString().endsWith("/")) {
2462                         baseUrl = new URL("%s%s".formatted(url, href));
2463                     }
2464                     else {
2465                         baseUrl = new URL(UrlUtils.resolveUrl(url, href));
2466                     }
2467                 }
2468                 catch (final MalformedURLException e) {
2469                     notifyIncorrectness("Invalid base url: \"" + href + "\", ignoring it");
2470                     baseUrl = url;
2471                 }
2472             }
2473         }
2474 
2475         return baseUrl;
2476     }
2477 
2478     /**
2479      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2480      *
2481      * Adds an {@link AutoCloseable}, which would be closed during the {@link #cleanUp()}.
2482      * @param autoCloseable the autoclosable
2483      */
2484     public void addAutoCloseable(final AutoCloseable autoCloseable) {
2485         if (autoCloseable == null) {
2486             return;
2487         }
2488 
2489         if (autoCloseableList_ == null) {
2490             autoCloseableList_ = new ArrayList<>();
2491         }
2492         autoCloseableList_.add(autoCloseable);
2493     }
2494 
2495     /**
2496      * {@inheritDoc}
2497      */
2498     @Override
2499     public boolean handles(final Event event) {
2500         if (Event.TYPE_BLUR.equals(event.getType()) || Event.TYPE_FOCUS.equals(event.getType())) {
2501             return true;
2502         }
2503         return super.handles(event);
2504     }
2505 
2506     /**
2507      * Sets the {@link ElementFromPointHandler}.
2508      * @param elementFromPointHandler the handler
2509      */
2510     public void setElementFromPointHandler(final ElementFromPointHandler elementFromPointHandler) {
2511         elementFromPointHandler_ = elementFromPointHandler;
2512     }
2513 
2514     /**
2515      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2516      *
2517      * Returns the element for the specified x coordinate and the specified y coordinate.
2518      *
2519      * @param x the x offset, in pixels
2520      * @param y the y offset, in pixels
2521      * @return the element for the specified x coordinate and the specified y coordinate
2522      */
2523     public HtmlElement getElementFromPoint(final int x, final int y) {
2524         if (elementFromPointHandler_ == null) {
2525             if (LOG.isWarnEnabled()) {
2526                 LOG.warn("ElementFromPointHandler was not specicifed for " + this);
2527             }
2528             if (x <= 0 || y <= 0) {
2529                 return null;
2530             }
2531             return getBody();
2532         }
2533         return elementFromPointHandler_.getElementFromPoint(this, x, y);
2534     }
2535 
2536     /**
2537      * Moves the focus to the specified element. This will trigger any relevant JavaScript
2538      * event handlers.
2539      *
2540      * @param newElement the element that will receive the focus, use {@code null} to remove focus from any element
2541      * @return true if the specified element now has the focus
2542      * @see #getFocusedElement()
2543      */
2544     public boolean setFocusedElement(final DomElement newElement) {
2545         return setFocusedElement(newElement, false);
2546     }
2547 
2548     /**
2549      * Moves the focus to the specified element. This will trigger any relevant JavaScript
2550      * event handlers.
2551      *
2552      * @param newElement the element that will receive the focus, use {@code null} to remove focus from any element
2553      * @param windowActivated - whether the enclosing window got focus resulting in specified element getting focus
2554      * @return true if the specified element now has the focus
2555      * @see #getFocusedElement()
2556      */
2557     public boolean setFocusedElement(final DomElement newElement, final boolean windowActivated) {
2558         if (elementWithFocus_ == newElement && !windowActivated) {
2559             // nothing to do
2560             return true;
2561         }
2562 
2563         final DomElement oldFocusedElement = elementWithFocus_;
2564         elementWithFocus_ = null;
2565 
2566         if (!windowActivated) {
2567             if (oldFocusedElement != null) {
2568                 oldFocusedElement.removeFocus();
2569                 oldFocusedElement.fireEvent(Event.TYPE_BLUR);
2570 
2571                 oldFocusedElement.fireEvent(Event.TYPE_FOCUS_OUT);
2572             }
2573         }
2574 
2575         elementWithFocus_ = newElement;
2576 
2577         // use newElement in the code below because element elementWithFocus_
2578         // might be changed by another thread
2579         if (newElement != null) {
2580             newElement.focus();
2581             newElement.fireEvent(Event.TYPE_FOCUS);
2582 
2583             newElement.fireEvent(Event.TYPE_FOCUS_IN);
2584         }
2585 
2586         // If a page reload happened as a result of the focus change then obviously this
2587         // element will not have the focus because its page has gone away.
2588         return this == getEnclosingWindow().getEnclosedPage();
2589     }
2590 
2591     /**
2592      * Returns the element with the focus or null if no element has the focus.
2593      * @return the element with focus or null
2594      * @see #setFocusedElement(DomElement)
2595      */
2596     public DomElement getFocusedElement() {
2597         return elementWithFocus_;
2598     }
2599 
2600     /**
2601      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2602      *
2603      * Sets the element with focus.
2604      * @param elementWithFocus the element with focus
2605      */
2606     public void setElementWithFocus(final DomElement elementWithFocus) {
2607         elementWithFocus_ = elementWithFocus;
2608     }
2609 
2610     /**
2611      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2612      * Returns the currently active element.
2613      *
2614      * @return the element that currently has focus, or the document's
2615      *         {@code body} element if no element has focus, or {@code null}
2616      *         if the document has no {@code body} element
2617      */
2618     public HtmlElement getActiveElement() {
2619         final DomElement activeElement = getFocusedElement();
2620         if (activeElement instanceof HtmlElement element) {
2621             return element;
2622         }
2623 
2624         final HtmlElement body = getBody();
2625         if (body != null) {
2626             return body;
2627         }
2628         return null;
2629     }
2630 
2631     /**
2632      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2633      *
2634      * <p>Returns the page's current selection ranges.</p>
2635      *
2636      * @return the page's current selection ranges
2637      */
2638     public List<SimpleRange> getSelectionRanges() {
2639         return selectionRanges_;
2640     }
2641 
2642     /**
2643      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2644      *
2645      * <p>Makes the specified selection range the *only* selection range on this page.</p>
2646      *
2647      * @param selectionRange the selection range
2648      */
2649     public void setSelectionRange(final SimpleRange selectionRange) {
2650         selectionRanges_.clear();
2651         selectionRanges_.add(selectionRange);
2652     }
2653 
2654     /**
2655      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2656      *
2657      * Execute a Function in the given context.
2658      *
2659      * @param function the JavaScript Function to call
2660      * @param thisObject the "this" object to be used during invocation
2661      * @param args the arguments to pass into the call
2662      * @param htmlElement the HTML element for which this script is being executed
2663      *        This element will be the context during the JavaScript execution. If null,
2664      *        the context will default to the page.
2665      * @return a ScriptResult which will contain both the current page (which may be different from
2666      *        the previous page) and a JavaScript result object.
2667      */
2668     public ScriptResult executeJavaScriptFunction(final Object function, final Object thisObject,
2669             final Object[] args, final DomNode htmlElement) {
2670         if (!getWebClient().isJavaScriptEnabled()) {
2671             return new ScriptResult(null);
2672         }
2673 
2674         final JavaScriptEngine engine = (JavaScriptEngine) getWebClient().getJavaScriptEngine();
2675         final Object result = engine.callFunction(this,
2676                                 (Function) function, (Scriptable) thisObject, args, htmlElement);
2677 
2678         return new ScriptResult(result);
2679     }
2680 
2681     private void writeObject(final ObjectOutputStream oos) throws IOException {
2682         oos.defaultWriteObject();
2683         oos.writeObject(originalCharset_ == null ? null : originalCharset_.name());
2684     }
2685 
2686     private void readObject(final ObjectInputStream ois) throws ClassNotFoundException, IOException {
2687         ois.defaultReadObject();
2688         final String charsetName = (String) ois.readObject();
2689         if (charsetName != null) {
2690             originalCharset_ = Charset.forName(charsetName);
2691         }
2692     }
2693 
2694     /**
2695      * {@inheritDoc}
2696      */
2697     @Override
2698     public void setNodeValue(final String value) {
2699         // Default behavior is to do nothing, overridden in some subclasses
2700     }
2701 
2702     /**
2703      * {@inheritDoc}
2704      */
2705     @Override
2706     public void setPrefix(final String prefix) {
2707         // Empty.
2708     }
2709 
2710     /**
2711      * {@inheritDoc}
2712      */
2713     @Override
2714     public void clearComputedStyles() {
2715         if (computedStylesCache_ != null) {
2716             computedStylesCache_.clear();
2717         }
2718     }
2719 
2720     /**
2721      * {@inheritDoc}
2722      */
2723     @Override
2724     public void clearComputedStyles(final DomElement element) {
2725         if (computedStylesCache_ != null) {
2726             computedStylesCache_.remove(element);
2727         }
2728     }
2729 
2730     /**
2731      * {@inheritDoc}
2732      */
2733     @Override
2734     public void clearComputedStylesUpToRoot(final DomElement element) {
2735         if (computedStylesCache_ != null) {
2736             computedStylesCache_.remove(element);
2737 
2738             DomNode parent = element.getParentNode();
2739             while (parent != null) {
2740                 computedStylesCache_.remove(parent);
2741                 parent = parent.getParentNode();
2742             }
2743         }
2744     }
2745 
2746     /**
2747      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2748      *
2749      * @param element the element to clear its cache
2750      * @param normalizedPseudo the pseudo attribute
2751      * @return the cached ComputedCssStyleDeclaration object or null
2752      */
2753     public ComputedCssStyleDeclaration getStyleFromCache(final DomElement element,
2754             final String normalizedPseudo) {
2755         return getCssPropertiesCache().get(element, normalizedPseudo);
2756     }
2757 
2758     /**
2759      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2760      *
2761      * Caches a ComputedCssStyleDeclaration object.
2762      * @param element the element to clear its cache
2763      * @param normalizedPseudo the pseudo attribute
2764      * @param style the ComputedCssStyleDeclaration to cache
2765      */
2766     public void putStyleIntoCache(final DomElement element, final String normalizedPseudo,
2767             final ComputedCssStyleDeclaration style) {
2768         getCssPropertiesCache().put(element, normalizedPseudo, style);
2769     }
2770 
2771     /**
2772      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2773      *
2774      * @return a list of all styles from this page (&lt;style&gt; and &lt;link rel=stylesheet&gt;).
2775      *         This returns an empty list if css support is disabled in the web client options.
2776      */
2777     public List<CssStyleSheet> getStyleSheets() {
2778         final List<CssStyleSheet> styles = new ArrayList<>();
2779         if (getWebClient().getOptions().isCssEnabled()) {
2780             for (final HtmlElement htmlElement : getHtmlElementDescendants()) {
2781                 if (htmlElement instanceof HtmlStyle style) {
2782                     styles.add(style.getSheet());
2783                     continue;
2784                 }
2785 
2786                 if (htmlElement instanceof HtmlLink link) {
2787                     if (link.isStyleSheetLink()) {
2788                         styles.add(link.getSheet());
2789                     }
2790                 }
2791             }
2792         }
2793         return styles;
2794     }
2795 
2796     /**
2797      * Returns the computed styles cache for this page.
2798      *
2799      * @return the computed styles cache for this page
2800      */
2801     private ComputedStylesCache getCssPropertiesCache() {
2802         if (computedStylesCache_ == null) {
2803             computedStylesCache_ = new ComputedStylesCache();
2804 
2805             // maintain the style cache
2806             final DomHtmlAttributeChangeListenerImpl listener = new DomHtmlAttributeChangeListenerImpl();
2807             addDomChangeListener(listener);
2808             addHtmlAttributeChangeListener(listener);
2809         }
2810         return computedStylesCache_;
2811     }
2812 
2813     /**
2814      * <p>Listens for changes anywhere in the document and evicts cached computed styles whenever something relevant
2815      * changes. Note that the very lazy way of doing this (completely clearing the cache every time something happens)
2816      * results in very meager performance gains. In order to get good (but still correct) performance, we need to be
2817      * a little smarter.</p>
2818      *
2819      * <p>CSS 2.1 has the following <a href="http://www.w3.org/TR/CSS21/selector.html">selector types</a> (where "SN" is
2820      * shorthand for "the selected node"):</p>
2821      *
2822      * <ol>
2823      *   <li><em>Universal</em> (i.e. "*"): Affected by the removal of SN from the document.</li>
2824      *   <li><em>Type</em> (i.e. "div"): Affected by the removal of SN from the document.</li>
2825      *   <li><em>Descendant</em> (i.e. "div span"): Affected by changes to SN or to any of its ancestors.</li>
2826      *   <li><em>Child</em> (i.e. "div &gt; span"): Affected by changes to SN or to its parent.</li>
2827      *   <li><em>Adjacent Sibling</em> (i.e. "table + p"): Affected by changes to SN or its previous sibling.</li>
2828      *   <li><em>Attribute</em> (i.e. "div.up, div[class~=up]"): Affected by changes to an attribute of SN.</li>
2829      *   <li><em>ID</em> (i.e. "#header"): Affected by changes to the <code>id</code> attribute of SN.</li>
2830      *   <li><em>Pseudo-Elements and Pseudo-Classes</em> (i.e. "p:first-child"): Affected by changes to parent.</li>
2831      * </ol>
2832      *
2833      * <p>Together, these rules dictate that the smart (but still lazy) way of removing elements from the computed style
2834      * cache is as follows -- whenever a node changes in any way, the cache needs to be cleared of styles for nodes
2835      * which:</p>
2836      *
2837      * <ul>
2838      *   <li>are actually the same node as the node that changed</li>
2839      *   <li>are siblings of the node that changed</li>
2840      *   <li>are descendants of the node that changed</li>
2841      * </ul>
2842      *
2843      * <p>Additionally, whenever a <code>style</code> node or a <code>link</code> node
2844      * with <code>rel=stylesheet</code> is added or
2845      * removed, all elements should be removed from the computed style cache.</p>
2846      */
2847     private class DomHtmlAttributeChangeListenerImpl implements DomChangeListener, HtmlAttributeChangeListener {
2848 
2849         /**
2850          * Ctor.
2851          */
2852         DomHtmlAttributeChangeListenerImpl() {
2853             super();
2854         }
2855 
2856         /**
2857          * {@inheritDoc}
2858          */
2859         @Override
2860         public void nodeAdded(final DomChangeEvent event) {
2861             nodeChanged(event.getChangedNode(), null);
2862         }
2863 
2864         /**
2865          * {@inheritDoc}
2866          */
2867         @Override
2868         public void nodeDeleted(final DomChangeEvent event) {
2869             nodeChanged(event.getChangedNode(), null);
2870         }
2871 
2872         /**
2873          * {@inheritDoc}
2874          */
2875         @Override
2876         public void attributeAdded(final HtmlAttributeChangeEvent event) {
2877             nodeChanged(event.getHtmlElement(), event.getName());
2878         }
2879 
2880         /**
2881          * {@inheritDoc}
2882          */
2883         @Override
2884         public void attributeRemoved(final HtmlAttributeChangeEvent event) {
2885             nodeChanged(event.getHtmlElement(), event.getName());
2886         }
2887 
2888         /**
2889          * {@inheritDoc}
2890          */
2891         @Override
2892         public void attributeReplaced(final HtmlAttributeChangeEvent event) {
2893             nodeChanged(event.getHtmlElement(), event.getName());
2894         }
2895 
2896         private void nodeChanged(final DomNode changedNode, final String attribName) {
2897             // If a stylesheet was changed, all of our calculations could be off; clear the cache.
2898             if (changedNode instanceof HtmlStyle) {
2899                 clearComputedStyles();
2900                 return;
2901             }
2902             if (changedNode instanceof HtmlLink link) {
2903                 if (link.isStyleSheetLink()) {
2904                     clearComputedStyles();
2905                     return;
2906                 }
2907             }
2908 
2909             // Apparently it wasn't a stylesheet that changed; be semi-smart about what we evict and when.
2910             // null means that a node was added/removed; we always have to take care of this for the parents
2911             final boolean clearParents = attribName == null || ATTRIBUTES_AFFECTING_PARENT.contains(attribName);
2912             if (computedStylesCache_ != null) {
2913                 computedStylesCache_.nodeChanged(changedNode, clearParents);
2914             }
2915         }
2916     }
2917 
2918     /**
2919      * Cache computed styles when possible, because their calculation is very expensive.
2920      * We use a weak hash map because we don't want this cache to be the only reason
2921      * nodes are kept around in the JVM, if all other references to them are gone.
2922      */
2923     private static final class ComputedStylesCache implements Serializable {
2924         private transient WeakHashMap<DomElement, Map<String, ComputedCssStyleDeclaration>>
2925                     computedStyles_ = new WeakHashMap<>();
2926 
2927         /**
2928          * Ctor.
2929          */
2930         ComputedStylesCache() {
2931             super();
2932         }
2933 
2934         public synchronized ComputedCssStyleDeclaration get(final DomElement element,
2935                 final String normalizedPseudo) {
2936             final Map<String, ComputedCssStyleDeclaration> elementMap = computedStyles_.get(element);
2937             if (elementMap != null) {
2938                 return elementMap.get(normalizedPseudo);
2939             }
2940             return null;
2941         }
2942 
2943         public synchronized void put(final DomElement element,
2944                 final String normalizedPseudo, final ComputedCssStyleDeclaration style) {
2945             final Map<String, ComputedCssStyleDeclaration>
2946                     elementMap = computedStyles_.computeIfAbsent(element, k -> new WeakHashMap<>());
2947             elementMap.put(normalizedPseudo, style);
2948         }
2949 
2950         public synchronized void nodeChanged(final DomNode changed, final boolean clearParents) {
2951             final Iterator<Map.Entry<DomElement, Map<String, ComputedCssStyleDeclaration>>>
2952                     i = computedStyles_.entrySet().iterator();
2953             while (i.hasNext()) {
2954                 final Map.Entry<DomElement, Map<String, ComputedCssStyleDeclaration>> entry = i.next();
2955                 final DomElement node = entry.getKey();
2956                 if (changed == node
2957                     || changed.getParentNode() == node.getParentNode()
2958                     || changed.isAncestorOf(node)
2959                     || clearParents && node.isAncestorOf(changed)) {
2960                     i.remove();
2961                 }
2962             }
2963 
2964             // maybe this is a better solution but I have to think a bit more about this
2965             //
2966             //            if (computedStyles_.isEmpty()) {
2967             //                return;
2968             //            }
2969             //
2970             //            // remove all siblings
2971             //            DomNode parent = changed.getParentNode();
2972             //            if (parent != null) {
2973             //                for (DomNode sibling : parent.getChildNodes()) {
2974             //                    computedStyles_.remove(sibling.getScriptableObject());
2975             //                }
2976             //
2977             //                if (clearParents) {
2978             //                    // remove all parents
2979             //                    while (parent != null) {
2980             //                        computedStyles_.remove(parent.getScriptableObject());
2981             //                        parent = parent.getParentNode();
2982             //                    }
2983             //                }
2984             //            }
2985             //
2986             //            // remove changed itself and all descendants
2987             //            computedStyles_.remove(changed.getScriptableObject());
2988             //            for (DomNode descendant : changed.getDescendants()) {
2989             //                computedStyles_.remove(descendant.getScriptableObject());
2990             //            }
2991         }
2992 
2993         public synchronized void clear() {
2994             computedStyles_.clear();
2995         }
2996 
2997         public synchronized Map<String, ComputedCssStyleDeclaration> remove(final DomNode element) {
2998             return computedStyles_.remove(element);
2999         }
3000 
3001         private void readObject(final ObjectInputStream in) throws IOException, ClassNotFoundException {
3002             in.defaultReadObject();
3003             computedStyles_ = new WeakHashMap<>();
3004         }
3005     }
3006 
3007     private static final class MappedElementIndexEntry implements Serializable {
3008         private final ArrayList<DomElement> elements_;
3009         private boolean sorted_;
3010 
3011         MappedElementIndexEntry() {
3012             // we do not expect to many elements having the same id/name
3013             elements_ = new ArrayList<>(2);
3014             sorted_ = true;
3015         }
3016 
3017         void add(final DomElement element) {
3018             if (elements_.indexOf(element) == -1) {
3019                 elements_.add(element);
3020                 sorted_ = elements_.size() < 2;
3021             }
3022         }
3023 
3024         DomElement first() {
3025             if (elements_.isEmpty()) {
3026                 return null;
3027             }
3028 
3029             if (sorted_) {
3030                 return elements_.get(0);
3031             }
3032 
3033             elements_.sort(DOCUMENT_POSITION_COMPERATOR);
3034             sorted_ = true;
3035 
3036             return elements_.get(0);
3037         }
3038 
3039         List<DomElement> elements() {
3040             if (sorted_) {
3041                 return elements_;
3042             }
3043 
3044             elements_.sort(DOCUMENT_POSITION_COMPERATOR);
3045             sorted_ = true;
3046 
3047             return elements_;
3048         }
3049 
3050         void remove(final DomElement element) {
3051             elements_.remove(element);
3052             sorted_ = elements_.size() < 2;
3053         }
3054     }
3055 }